[mirror_ubuntu-bionic-kernel.git] / drivers / base / dma-coherent.c

/*
 * Coherent per-device memory handling.
 * Borrowed from i386
 */
#include <linux/io.h>
#include <linux/slab.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/dma-mapping.h>

struct dma_coherent_mem {
	void		*virt_base;
	dma_addr_t	device_base;
	unsigned long	pfn_base;
	int		size;
	int		flags;
	unsigned long	*bitmap;
	spinlock_t	spinlock;
	bool		use_dev_dma_pfn_offset;
};

static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init;

static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *dev)
{
	if (dev && dev->dma_mem)
		return dev->dma_mem;
	return NULL;
}

static inline dma_addr_t dma_get_device_base(struct device *dev,
					     struct dma_coherent_mem * mem)
{
	if (mem->use_dev_dma_pfn_offset)
		return (mem->pfn_base - dev->dma_pfn_offset) << PAGE_SHIFT;
	else
		return mem->device_base;
}

static int dma_init_coherent_memory(
	phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags,
	struct dma_coherent_mem **mem)
{
	struct dma_coherent_mem *dma_mem = NULL;
	void __iomem *mem_base = NULL;
	int pages = size >> PAGE_SHIFT;
	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
	int ret;

	if (!size) {
		ret = -EINVAL;
		goto out;
	}

	mem_base = memremap(phys_addr, size, MEMREMAP_WC);
	if (!mem_base) {
		ret = -EINVAL;
		goto out;
	}
	dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
	if (!dma_mem) {
		ret = -ENOMEM;
		goto out;
	}
	dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
	if (!dma_mem->bitmap) {
		ret = -ENOMEM;
		goto out;
	}

	dma_mem->virt_base = mem_base;
	dma_mem->device_base = device_addr;
	dma_mem->pfn_base = PFN_DOWN(phys_addr);
	dma_mem->size = pages;
	dma_mem->flags = flags;
	spin_lock_init(&dma_mem->spinlock);

	*mem = dma_mem;
	return 0;

out:
	kfree(dma_mem);
	if (mem_base)
		memunmap(mem_base);
	return ret;
}

static void dma_release_coherent_memory(struct dma_coherent_mem *mem)
{
	if (!mem)
		return;

	memunmap(mem->virt_base);
	kfree(mem->bitmap);
	kfree(mem);
}

static int dma_assign_coherent_memory(struct device *dev,
				      struct dma_coherent_mem *mem)
{
	if (!dev)
		return -ENODEV;

	if (dev->dma_mem)
		return -EBUSY;

	dev->dma_mem = mem;
	return 0;
}

int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
				dma_addr_t device_addr, size_t size, int flags)
{
	struct dma_coherent_mem *mem;
	int ret;

	ret = dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem);
	if (ret)
		return ret;

	ret = dma_assign_coherent_memory(dev, mem);
	if (ret)
		dma_release_coherent_memory(mem);
	return ret;
}
EXPORT_SYMBOL(dma_declare_coherent_memory);

void dma_release_declared_memory(struct device *dev)
{
	struct dma_coherent_mem *mem = dev->dma_mem;

	if (!mem)
		return;
	dma_release_coherent_memory(mem);
	dev->dma_mem = NULL;
}
EXPORT_SYMBOL(dma_release_declared_memory);

void *dma_mark_declared_memory_occupied(struct device *dev,
					dma_addr_t device_addr, size_t size)
{
	struct dma_coherent_mem *mem = dev->dma_mem;
	unsigned long flags;
	int pos, err;

	size += device_addr & ~PAGE_MASK;

	if (!mem)
		return ERR_PTR(-EINVAL);

	spin_lock_irqsave(&mem->spinlock, flags);
	pos = PFN_DOWN(device_addr - dma_get_device_base(dev, mem));
	err = bitmap_allocate_region(mem->bitmap, pos, get_order(size));
	spin_unlock_irqrestore(&mem->spinlock, flags);

	if (err != 0)
		return ERR_PTR(err);
	return mem->virt_base + (pos << PAGE_SHIFT);
}
EXPORT_SYMBOL(dma_mark_declared_memory_occupied);

static void *__dma_alloc_from_coherent(struct dma_coherent_mem *mem,
		ssize_t size, dma_addr_t *dma_handle)
{
	int order = get_order(size);
	unsigned long flags;
	int pageno;
	void *ret;

	spin_lock_irqsave(&mem->spinlock, flags);

	if (unlikely(size > (mem->size << PAGE_SHIFT)))
		goto err;

	pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);
	if (unlikely(pageno < 0))
		goto err;

	/*
	 * Memory was found in the coherent area.
	 */
	*dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
	ret = mem->virt_base + (pageno << PAGE_SHIFT);
	spin_unlock_irqrestore(&mem->spinlock, flags);
	memset(ret, 0, size);
	return ret;
err:
	spin_unlock_irqrestore(&mem->spinlock, flags);
	return NULL;
}

/**
 * dma_alloc_from_dev_coherent() - allocate memory from device coherent pool
 * @dev:	device from which we allocate memory
 * @size:	size of requested memory area
 * @dma_handle:	This will be filled with the correct dma handle
 * @ret:	This pointer will be filled with the virtual address
 *		to allocated area.
 *
 * This function should be only called from per-arch dma_alloc_coherent()
 * to support allocation from per-device coherent memory pools.
 *
 * Returns 0 if dma_alloc_coherent should continue with allocating from
 * generic memory areas, or !0 if dma_alloc_coherent should return @ret.
 */
int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
		dma_addr_t *dma_handle, void **ret)
{
	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);

	if (!mem)
		return 0;

	*ret = __dma_alloc_from_coherent(mem, size, dma_handle);
	if (*ret)
		return 1;

	/*
	 * In the case where the allocation can not be satisfied from the
	 * per-device area, try to fall back to generic memory if the
	 * constraints allow it.
	 */
	return mem->flags & DMA_MEMORY_EXCLUSIVE;
}
EXPORT_SYMBOL(dma_alloc_from_dev_coherent);

void *dma_alloc_from_global_coherent(ssize_t size, dma_addr_t *dma_handle)
{
	if (!dma_coherent_default_memory)
		return NULL;

	return __dma_alloc_from_coherent(dma_coherent_default_memory, size,
			dma_handle);
}

static int __dma_release_from_coherent(struct dma_coherent_mem *mem,
				       int order, void *vaddr)
{
	if (mem && vaddr >= mem->virt_base && vaddr <
		   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
		unsigned long flags;

		spin_lock_irqsave(&mem->spinlock, flags);
		bitmap_release_region(mem->bitmap, page, order);
		spin_unlock_irqrestore(&mem->spinlock, flags);
		return 1;
	}
	return 0;
}

/**
 * dma_release_from_dev_coherent() - free memory to device coherent memory pool
 * @dev:	device from which the memory was allocated
 * @order:	the order of pages allocated
 * @vaddr:	virtual address of allocated pages
 *
 * This checks whether the memory was allocated from the per-device
 * coherent memory pool and if so, releases that memory.
 *
 * Returns 1 if we correctly released the memory, or 0 if the caller should
 * proceed with releasing memory from generic pools.
 */
int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr)
{
	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);

	return __dma_release_from_coherent(mem, order, vaddr);
}
EXPORT_SYMBOL(dma_release_from_dev_coherent);

int dma_release_from_global_coherent(int order, void *vaddr)
{
	if (!dma_coherent_default_memory)
		return 0;

	return __dma_release_from_coherent(dma_coherent_default_memory, order,
			vaddr);
}

static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,
		struct vm_area_struct *vma, void *vaddr, size_t size, int *ret)
{
	if (mem && vaddr >= mem->virt_base && vaddr + size <=
		   (mem->virt_base + (mem->size << PAGE_SHIFT))) {
		unsigned long off = vma->vm_pgoff;
		int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;
		int user_count = vma_pages(vma);
		int count = PAGE_ALIGN(size) >> PAGE_SHIFT;

		*ret = -ENXIO;
		if (off < count && user_count <= count - off) {
			unsigned long pfn = mem->pfn_base + start + off;
			*ret = remap_pfn_range(vma, vma->vm_start, pfn,
					       user_count << PAGE_SHIFT,
					       vma->vm_page_prot);
		}
		return 1;
	}
	return 0;
}

/**
 * dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool
 * @dev:	device from which the memory was allocated
 * @vma:	vm_area for the userspace memory
 * @vaddr:	cpu address returned by dma_alloc_from_dev_coherent
 * @size:	size of the memory buffer allocated
 * @ret:	result from remap_pfn_range()
 *
 * This checks whether the memory was allocated from the per-device
 * coherent memory pool and if so, maps that memory to the provided vma.
 *
 * Returns 1 if we correctly mapped the memory, or 0 if the caller should
 * proceed with mapping memory from generic pools.
 */
int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma,
			   void *vaddr, size_t size, int *ret)
{
	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);

	return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);
}
EXPORT_SYMBOL(dma_mmap_from_dev_coherent);

int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *vaddr,
				   size_t size, int *ret)
{
	if (!dma_coherent_default_memory)
		return 0;

	return __dma_mmap_from_coherent(dma_coherent_default_memory, vma,
					vaddr, size, ret);
}

/*
 * Support for reserved memory regions defined in device tree
 */
#ifdef CONFIG_OF_RESERVED_MEM
#include <linux/of.h>
#include <linux/of_fdt.h>
#include <linux/of_reserved_mem.h>

static struct reserved_mem *dma_reserved_default_memory __initdata;

static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev)
{
	struct dma_coherent_mem *mem = rmem->priv;
	int ret;

	if (!mem) {
		ret = dma_init_coherent_memory(rmem->base, rmem->base,
					       rmem->size,
					       DMA_MEMORY_EXCLUSIVE, &mem);
		if (ret) {
			pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
				&rmem->base, (unsigned long)rmem->size / SZ_1M);
			return ret;
		}
	}
	mem->use_dev_dma_pfn_offset = true;
	rmem->priv = mem;
	dma_assign_coherent_memory(dev, mem);
	return 0;
}

static void rmem_dma_device_release(struct reserved_mem *rmem,
				    struct device *dev)
{
	if (dev)
		dev->dma_mem = NULL;
}

static const struct reserved_mem_ops rmem_dma_ops = {
	.device_init	= rmem_dma_device_init,
	.device_release	= rmem_dma_device_release,
};

static int __init rmem_dma_setup(struct reserved_mem *rmem)
{
	unsigned long node = rmem->fdt_node;

	if (of_get_flat_dt_prop(node, "reusable", NULL))
		return -EINVAL;

#ifdef CONFIG_ARM
	if (!of_get_flat_dt_prop(node, "no-map", NULL)) {
		pr_err("Reserved memory: regions without no-map are not yet supported\n");
		return -EINVAL;
	}

	if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {
		WARN(dma_reserved_default_memory,
		     "Reserved memory: region for default DMA coherent area is redefined\n");
		dma_reserved_default_memory = rmem;
	}
#endif

	rmem->ops = &rmem_dma_ops;
	pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n",
		&rmem->base, (unsigned long)rmem->size / SZ_1M);
	return 0;
}

static int __init dma_init_reserved_memory(void)
{
	const struct reserved_mem_ops *ops;
	int ret;

	if (!dma_reserved_default_memory)
		return -ENOMEM;

	ops = dma_reserved_default_memory->ops;

	/*
	 * We rely on rmem_dma_device_init() does not propagate error of
	 * dma_assign_coherent_memory() for "NULL" device.
	 */
	ret = ops->device_init(dma_reserved_default_memory, NULL);

	if (!ret) {
		dma_coherent_default_memory = dma_reserved_default_memory->priv;
		pr_info("DMA: default coherent area is set\n");
	}

	return ret;
}

core_initcall(dma_init_reserved_memory);

RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);
#endif
Commit	Line	Data
ee7e5516 DES	1	/*
	2	* Coherent per-device memory handling.
	3	* Borrowed from i386
	4	*/
6b03ae0d	5	#include <linux/io.h>
5a0e3ad6	6	#include <linux/slab.h>
ee7e5516	7	#include <linux/kernel.h>
08a999ce	8	#include <linux/module.h>
ee7e5516 DES	9	#include <linux/dma-mapping.h>
	10
	11	struct dma_coherent_mem {
	12	void *virt_base;
ed1d218c	13	dma_addr_t device_base;
88a984ba	14	unsigned long pfn_base;
ee7e5516 DES	15	int size;
	16	int flags;
	17	unsigned long *bitmap;
7bfa5ab6	18	spinlock_t spinlock;
c41f9ea9	19	bool use_dev_dma_pfn_offset;
ee7e5516 DES	20	};
ee7e5516 DES	21
93228b44 VM	22	static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init;
	23
	24	static inline struct dma_coherent_mem dev_get_coherent_memory(struct device dev)
	25	{
	26	if (dev && dev->dma_mem)
	27	return dev->dma_mem;
43fc509c	28	return NULL;
93228b44 VM	29	}
93228b44 VM	30
c41f9ea9 VM	31	static inline dma_addr_t dma_get_device_base(struct device *dev,
	32	struct dma_coherent_mem * mem)
	33	{
	34	if (mem->use_dev_dma_pfn_offset)
	35	return (mem->pfn_base - dev->dma_pfn_offset) << PAGE_SHIFT;
	36	else
	37	return mem->device_base;
	38	}
	39
d35b0996	40	static int dma_init_coherent_memory(
9e5b3d6f MN	41	phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags,
9e5b3d6f MN	42	struct dma_coherent_mem **mem)
ee7e5516	43	{
7bfa5ab6	44	struct dma_coherent_mem *dma_mem = NULL;
ee7e5516 DES	45	void __iomem *mem_base = NULL;
	46	int pages = size >> PAGE_SHIFT;
	47	int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
d35b0996	48	int ret;
ee7e5516	49
d35b0996 AB	50	if (!size) {
d35b0996 AB	51	ret = -EINVAL;
ee7e5516	52	goto out;
d35b0996	53	}
ee7e5516	54
2436bdcd	55	mem_base = memremap(phys_addr, size, MEMREMAP_WC);
d35b0996 AB	56	if (!mem_base) {
d35b0996 AB	57	ret = -EINVAL;
ee7e5516	58	goto out;
d35b0996	59	}
7bfa5ab6	60	dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
d35b0996 AB	61	if (!dma_mem) {
d35b0996 AB	62	ret = -ENOMEM;
ee7e5516	63	goto out;
d35b0996	64	}
7bfa5ab6	65	dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
d35b0996 AB	66	if (!dma_mem->bitmap) {
d35b0996 AB	67	ret = -ENOMEM;
7bfa5ab6	68	goto out;
d35b0996	69	}
7bfa5ab6 MS	70
	71	dma_mem->virt_base = mem_base;
	72	dma_mem->device_base = device_addr;
	73	dma_mem->pfn_base = PFN_DOWN(phys_addr);
	74	dma_mem->size = pages;
	75	dma_mem->flags = flags;
	76	spin_lock_init(&dma_mem->spinlock);
ee7e5516	77
7bfa5ab6	78	*mem = dma_mem;
d35b0996	79	return 0;
ee7e5516	80
7bfa5ab6 MS	81	out:
7bfa5ab6 MS	82	kfree(dma_mem);
2436bdcd CH	83	if (mem_base)
2436bdcd CH	84	memunmap(mem_base);
d35b0996	85	return ret;
ee7e5516	86	}
7bfa5ab6 MS	87
	88	static void dma_release_coherent_memory(struct dma_coherent_mem *mem)
	89	{
	90	if (!mem)
	91	return;
6b03ae0d	92
2436bdcd	93	memunmap(mem->virt_base);
7bfa5ab6 MS	94	kfree(mem->bitmap);
	95	kfree(mem);
	96	}
	97
	98	static int dma_assign_coherent_memory(struct device *dev,
	99	struct dma_coherent_mem *mem)
	100	{
93228b44 VM	101	if (!dev)
	102	return -ENODEV;
	103
7bfa5ab6 MS	104	if (dev->dma_mem)
	105	return -EBUSY;
	106
	107	dev->dma_mem = mem;
7bfa5ab6 MS	108	return 0;
	109	}
	110
	111	int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr,
	112	dma_addr_t device_addr, size_t size, int flags)
	113	{
	114	struct dma_coherent_mem *mem;
2436bdcd	115	int ret;
7bfa5ab6	116
2436bdcd CH	117	ret = dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem);
	118	if (ret)
	119	return ret;
7bfa5ab6	120
2436bdcd CH	121	ret = dma_assign_coherent_memory(dev, mem);
	122	if (ret)
	123	dma_release_coherent_memory(mem);
	124	return ret;
7bfa5ab6	125	}
ee7e5516 DES	126	EXPORT_SYMBOL(dma_declare_coherent_memory);
	127
	128	void dma_release_declared_memory(struct device *dev)
	129	{
	130	struct dma_coherent_mem *mem = dev->dma_mem;
	131
	132	if (!mem)
	133	return;
7bfa5ab6	134	dma_release_coherent_memory(mem);
ee7e5516	135	dev->dma_mem = NULL;
ee7e5516 DES	136	}
	137	EXPORT_SYMBOL(dma_release_declared_memory);
	138
	139	void dma_mark_declared_memory_occupied(struct device dev,
	140	dma_addr_t device_addr, size_t size)
	141	{
	142	struct dma_coherent_mem *mem = dev->dma_mem;
7bfa5ab6	143	unsigned long flags;
ee7e5516	144	int pos, err;
ee7e5516	145
d2dc1f4a	146	size += device_addr & ~PAGE_MASK;
ee7e5516 DES	147
	148	if (!mem)
	149	return ERR_PTR(-EINVAL);
	150
7bfa5ab6	151	spin_lock_irqsave(&mem->spinlock, flags);
c41f9ea9	152	pos = PFN_DOWN(device_addr - dma_get_device_base(dev, mem));
d2dc1f4a	153	err = bitmap_allocate_region(mem->bitmap, pos, get_order(size));
7bfa5ab6 MS	154	spin_unlock_irqrestore(&mem->spinlock, flags);
7bfa5ab6 MS	155
ee7e5516 DES	156	if (err != 0)
	157	return ERR_PTR(err);
	158	return mem->virt_base + (pos << PAGE_SHIFT);
	159	}
	160	EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
	161
43fc509c VM	162	static void __dma_alloc_from_coherent(struct dma_coherent_mem mem,
43fc509c VM	163	ssize_t size, dma_addr_t *dma_handle)
ee7e5516	164	{
ee7e5516	165	int order = get_order(size);
7bfa5ab6	166	unsigned long flags;
eccd83e1	167	int pageno;
43fc509c	168	void *ret;
ee7e5516	169
7bfa5ab6	170	spin_lock_irqsave(&mem->spinlock, flags);
0609697e	171
cdf57cab	172	if (unlikely(size > (mem->size << PAGE_SHIFT)))
0609697e	173	goto err;
eccd83e1 AM	174
eccd83e1 AM	175	pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);
0609697e PM	176	if (unlikely(pageno < 0))
	177	goto err;
	178
	179	/*
43fc509c	180	* Memory was found in the coherent area.
0609697e	181	*/
43fc509c VM	182	*dma_handle = mem->device_base + (pageno << PAGE_SHIFT);
43fc509c VM	183	ret = mem->virt_base + (pageno << PAGE_SHIFT);
dd01c75f	184	spin_unlock_irqrestore(&mem->spinlock, flags);
2436bdcd	185	memset(ret, 0, size);
43fc509c	186	return ret;
0609697e	187	err:
7bfa5ab6	188	spin_unlock_irqrestore(&mem->spinlock, flags);
43fc509c VM	189	return NULL;
	190	}
	191
	192	/**
	193	* dma_alloc_from_dev_coherent() - allocate memory from device coherent pool
	194	* @dev: device from which we allocate memory
	195	* @size: size of requested memory area
	196	* @dma_handle: This will be filled with the correct dma handle
	197	* @ret: This pointer will be filled with the virtual address
	198	* to allocated area.
	199	*
	200	* This function should be only called from per-arch dma_alloc_coherent()
	201	* to support allocation from per-device coherent memory pools.
	202	*
	203	* Returns 0 if dma_alloc_coherent should continue with allocating from
	204	* generic memory areas, or !0 if dma_alloc_coherent should return @ret.
	205	*/
	206	int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size,
	207	dma_addr_t dma_handle, void *ret)
	208	{
	209	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);
	210
	211	if (!mem)
	212	return 0;
	213
	214	*ret = __dma_alloc_from_coherent(mem, size, dma_handle);
	215	if (*ret)
	216	return 1;
	217
0609697e PM	218	/*
	219	* In the case where the allocation can not be satisfied from the
	220	* per-device area, try to fall back to generic memory if the
	221	* constraints allow it.
	222	*/
	223	return mem->flags & DMA_MEMORY_EXCLUSIVE;
ee7e5516	224	}
43fc509c	225	EXPORT_SYMBOL(dma_alloc_from_dev_coherent);
ee7e5516	226
43fc509c	227	void dma_alloc_from_global_coherent(ssize_t size, dma_addr_t dma_handle)
ee7e5516	228	{
43fc509c VM	229	if (!dma_coherent_default_memory)
	230	return NULL;
	231
	232	return __dma_alloc_from_coherent(dma_coherent_default_memory, size,
	233	dma_handle);
	234	}
ee7e5516	235
43fc509c VM	236	static int __dma_release_from_coherent(struct dma_coherent_mem *mem,
	237	int order, void *vaddr)
	238	{
ee7e5516 DES	239	if (mem && vaddr >= mem->virt_base && vaddr <
	240	(mem->virt_base + (mem->size << PAGE_SHIFT))) {
	241	int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
7bfa5ab6	242	unsigned long flags;
ee7e5516	243
7bfa5ab6	244	spin_lock_irqsave(&mem->spinlock, flags);
ee7e5516	245	bitmap_release_region(mem->bitmap, page, order);
7bfa5ab6	246	spin_unlock_irqrestore(&mem->spinlock, flags);
ee7e5516 DES	247	return 1;
	248	}
	249	return 0;
	250	}
bca0fa5f MS	251
bca0fa5f MS	252	/**
43fc509c	253	* dma_release_from_dev_coherent() - free memory to device coherent memory pool
bca0fa5f	254	* @dev: device from which the memory was allocated
43fc509c VM	255	* @order: the order of pages allocated
43fc509c VM	256	* @vaddr: virtual address of allocated pages
bca0fa5f MS	257	*
bca0fa5f MS	258	* This checks whether the memory was allocated from the per-device
43fc509c	259	* coherent memory pool and if so, releases that memory.
bca0fa5f	260	*
43fc509c VM	261	* Returns 1 if we correctly released the memory, or 0 if the caller should
43fc509c VM	262	* proceed with releasing memory from generic pools.
bca0fa5f	263	*/
43fc509c	264	int dma_release_from_dev_coherent(struct device dev, int order, void vaddr)
bca0fa5f	265	{
93228b44	266	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);
bca0fa5f	267
43fc509c VM	268	return __dma_release_from_coherent(mem, order, vaddr);
	269	}
	270	EXPORT_SYMBOL(dma_release_from_dev_coherent);
	271
	272	int dma_release_from_global_coherent(int order, void *vaddr)
	273	{
	274	if (!dma_coherent_default_memory)
	275	return 0;
	276
	277	return __dma_release_from_coherent(dma_coherent_default_memory, order,
	278	vaddr);
	279	}
	280
	281	static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,
	282	struct vm_area_struct vma, void vaddr, size_t size, int *ret)
	283	{
bca0fa5f MS	284	if (mem && vaddr >= mem->virt_base && vaddr + size <=
	285	(mem->virt_base + (mem->size << PAGE_SHIFT))) {
	286	unsigned long off = vma->vm_pgoff;
	287	int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;
e688f144	288	int user_count = vma_pages(vma);
9ca5d4fd	289	int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
bca0fa5f MS	290
	291	*ret = -ENXIO;
	292	if (off < count && user_count <= count - off) {
88a984ba	293	unsigned long pfn = mem->pfn_base + start + off;
bca0fa5f MS	294	*ret = remap_pfn_range(vma, vma->vm_start, pfn,
	295	user_count << PAGE_SHIFT,
	296	vma->vm_page_prot);
	297	}
	298	return 1;
	299	}
	300	return 0;
	301	}
43fc509c VM	302
	303	/**
	304	* dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool
	305	* @dev: device from which the memory was allocated
	306	* @vma: vm_area for the userspace memory
	307	* @vaddr: cpu address returned by dma_alloc_from_dev_coherent
	308	* @size: size of the memory buffer allocated
	309	* @ret: result from remap_pfn_range()
	310	*
	311	* This checks whether the memory was allocated from the per-device
	312	* coherent memory pool and if so, maps that memory to the provided vma.
	313	*
	314	* Returns 1 if we correctly mapped the memory, or 0 if the caller should
	315	* proceed with mapping memory from generic pools.
	316	*/
	317	int dma_mmap_from_dev_coherent(struct device dev, struct vm_area_struct vma,
	318	void vaddr, size_t size, int ret)
	319	{
	320	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);
	321
	322	return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);
	323	}
	324	EXPORT_SYMBOL(dma_mmap_from_dev_coherent);
	325
	326	int dma_mmap_from_global_coherent(struct vm_area_struct vma, void vaddr,
	327	size_t size, int *ret)
	328	{
	329	if (!dma_coherent_default_memory)
	330	return 0;
	331
	332	return __dma_mmap_from_coherent(dma_coherent_default_memory, vma,
	333	vaddr, size, ret);
	334	}
7bfa5ab6 MS	335
	336	/*
	337	* Support for reserved memory regions defined in device tree
	338	*/
	339	#ifdef CONFIG_OF_RESERVED_MEM
	340	#include <linux/of.h>
	341	#include <linux/of_fdt.h>
	342	#include <linux/of_reserved_mem.h>
	343
93228b44 VM	344	static struct reserved_mem *dma_reserved_default_memory __initdata;
93228b44 VM	345
7bfa5ab6 MS	346	static int rmem_dma_device_init(struct reserved_mem rmem, struct device dev)
	347	{
	348	struct dma_coherent_mem *mem = rmem->priv;
d35b0996 AB	349	int ret;
d35b0996 AB	350
6d573398 AB	351	if (!mem) {
	352	ret = dma_init_coherent_memory(rmem->base, rmem->base,
	353	rmem->size,
	354	DMA_MEMORY_EXCLUSIVE, &mem);
	355	if (ret) {
	356	pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
	357	&rmem->base, (unsigned long)rmem->size / SZ_1M);
	358	return ret;
	359	}
7bfa5ab6	360	}
c41f9ea9	361	mem->use_dev_dma_pfn_offset = true;
7bfa5ab6 MS	362	rmem->priv = mem;
	363	dma_assign_coherent_memory(dev, mem);
	364	return 0;
	365	}
	366
	367	static void rmem_dma_device_release(struct reserved_mem *rmem,
	368	struct device *dev)
	369	{
93228b44 VM	370	if (dev)
93228b44 VM	371	dev->dma_mem = NULL;
7bfa5ab6 MS	372	}
	373
	374	static const struct reserved_mem_ops rmem_dma_ops = {
	375	.device_init = rmem_dma_device_init,
	376	.device_release = rmem_dma_device_release,
	377	};
	378
	379	static int __init rmem_dma_setup(struct reserved_mem *rmem)
	380	{
	381	unsigned long node = rmem->fdt_node;
	382
	383	if (of_get_flat_dt_prop(node, "reusable", NULL))
	384	return -EINVAL;
	385
	386	#ifdef CONFIG_ARM
	387	if (!of_get_flat_dt_prop(node, "no-map", NULL)) {
	388	pr_err("Reserved memory: regions without no-map are not yet supported\n");
	389	return -EINVAL;
	390	}
93228b44 VM	391
	392	if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {
	393	WARN(dma_reserved_default_memory,
	394	"Reserved memory: region for default DMA coherent area is redefined\n");
	395	dma_reserved_default_memory = rmem;
	396	}
7bfa5ab6 MS	397	#endif
	398
	399	rmem->ops = &rmem_dma_ops;
	400	pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n",
	401	&rmem->base, (unsigned long)rmem->size / SZ_1M);
	402	return 0;
	403	}
93228b44 VM	404
	405	static int __init dma_init_reserved_memory(void)
	406	{
	407	const struct reserved_mem_ops *ops;
	408	int ret;
	409
	410	if (!dma_reserved_default_memory)
	411	return -ENOMEM;
	412
	413	ops = dma_reserved_default_memory->ops;
	414
	415	/*
	416	* We rely on rmem_dma_device_init() does not propagate error of
	417	* dma_assign_coherent_memory() for "NULL" device.
	418	*/
	419	ret = ops->device_init(dma_reserved_default_memory, NULL);
	420
	421	if (!ret) {
	422	dma_coherent_default_memory = dma_reserved_default_memory->priv;
	423	pr_info("DMA: default coherent area is set\n");
	424	}
	425
	426	return ret;
	427	}
	428
	429	core_initcall(dma_init_reserved_memory);
	430
7bfa5ab6 MS	431	RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);
7bfa5ab6 MS	432	#endif