[mirror_ubuntu-eoan-kernel.git] / arch / mips / mm / dma-default.c

/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 2000  Ani Joshi <ajoshi@unixbox.com>
 * Copyright (C) 2000, 2001, 06	 Ralf Baechle <ralf@linux-mips.org>
 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
 */

#include <linux/types.h>
#include <linux/dma-mapping.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/scatterlist.h>
#include <linux/string.h>
#include <linux/gfp.h>
#include <linux/highmem.h>

#include <asm/cache.h>
#include <asm/cpu-type.h>
#include <asm/io.h>

#include <dma-coherence.h>

int coherentio = 0;	/* User defined DMA coherency from command line. */
EXPORT_SYMBOL_GPL(coherentio);
int hw_coherentio = 0;	/* Actual hardware supported DMA coherency setting. */

static int __init setcoherentio(char *str)
{
	coherentio = 1;
	pr_info("Hardware DMA cache coherency (command line)\n");
	return 0;
}
early_param("coherentio", setcoherentio);

static int __init setnocoherentio(char *str)
{
	coherentio = 0;
	pr_info("Software DMA cache coherency (command line)\n");
	return 0;
}
early_param("nocoherentio", setnocoherentio);

static inline struct page *dma_addr_to_page(struct device *dev,
	dma_addr_t dma_addr)
{
	return pfn_to_page(
		plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
}

/*
 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
 * speculatively fill random cachelines with stale data at any time,
 * requiring an extra flush post-DMA.
 *
 * Warning on the terminology - Linux calls an uncached area coherent;
 * MIPS terminology calls memory areas with hardware maintained coherency
 * coherent.
 */
static inline int cpu_needs_post_dma_flush(struct device *dev)
{
	return !plat_device_is_coherent(dev) &&
	       (boot_cpu_type() == CPU_R10000 ||
		boot_cpu_type() == CPU_R12000 ||
		boot_cpu_type() == CPU_BMIPS5000);
}

static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
{
	gfp_t dma_flag;

	/* ignore region specifiers */
	gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);

#ifdef CONFIG_ISA
	if (dev == NULL)
		dma_flag = __GFP_DMA;
	else
#endif
#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
			dma_flag = __GFP_DMA;
	else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
			dma_flag = __GFP_DMA32;
	else
#endif
#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
		dma_flag = __GFP_DMA32;
	else
#endif
#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
	     if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
		dma_flag = __GFP_DMA;
	else
#endif
		dma_flag = 0;

	/* Don't invoke OOM killer */
	gfp |= __GFP_NORETRY;

	return gfp | dma_flag;
}

void *dma_alloc_noncoherent(struct device *dev, size_t size,
	dma_addr_t * dma_handle, gfp_t gfp)
{
	void *ret;

	gfp = massage_gfp_flags(dev, gfp);

	ret = (void *) __get_free_pages(gfp, get_order(size));

	if (ret != NULL) {
		memset(ret, 0, size);
		*dma_handle = plat_map_dma_mem(dev, ret, size);
	}

	return ret;
}
EXPORT_SYMBOL(dma_alloc_noncoherent);

static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
	dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
{
	void *ret;

	if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
		return ret;

	gfp = massage_gfp_flags(dev, gfp);

	ret = (void *) __get_free_pages(gfp, get_order(size));

	if (ret) {
		memset(ret, 0, size);
		*dma_handle = plat_map_dma_mem(dev, ret, size);

		if (!plat_device_is_coherent(dev)) {
			dma_cache_wback_inv((unsigned long) ret, size);
			if (!hw_coherentio)
				ret = UNCAC_ADDR(ret);
		}
	}

	return ret;
}


void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
	dma_addr_t dma_handle)
{
	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
	free_pages((unsigned long) vaddr, get_order(size));
}
EXPORT_SYMBOL(dma_free_noncoherent);

static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
	dma_addr_t dma_handle, struct dma_attrs *attrs)
{
	unsigned long addr = (unsigned long) vaddr;
	int order = get_order(size);

	if (dma_release_from_coherent(dev, order, vaddr))
		return;

	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);

	if (!plat_device_is_coherent(dev) && !hw_coherentio)
		addr = CAC_ADDR(addr);

	free_pages(addr, get_order(size));
}

static inline void __dma_sync_virtual(void *addr, size_t size,
	enum dma_data_direction direction)
{
	switch (direction) {
	case DMA_TO_DEVICE:
		dma_cache_wback((unsigned long)addr, size);
		break;

	case DMA_FROM_DEVICE:
		dma_cache_inv((unsigned long)addr, size);
		break;

	case DMA_BIDIRECTIONAL:
		dma_cache_wback_inv((unsigned long)addr, size);
		break;

	default:
		BUG();
	}
}

/*
 * A single sg entry may refer to multiple physically contiguous
 * pages. But we still need to process highmem pages individually.
 * If highmem is not configured then the bulk of this loop gets
 * optimized out.
 */
static inline void __dma_sync(struct page *page,
	unsigned long offset, size_t size, enum dma_data_direction direction)
{
	size_t left = size;

	do {
		size_t len = left;

		if (PageHighMem(page)) {
			void *addr;

			if (offset + len > PAGE_SIZE) {
				if (offset >= PAGE_SIZE) {
					page += offset >> PAGE_SHIFT;
					offset &= ~PAGE_MASK;
				}
				len = PAGE_SIZE - offset;
			}

			addr = kmap_atomic(page);
			__dma_sync_virtual(addr + offset, len, direction);
			kunmap_atomic(addr);
		} else
			__dma_sync_virtual(page_address(page) + offset,
					   size, direction);
		offset = 0;
		page++;
		left -= len;
	} while (left);
}

static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
	size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
{
	if (cpu_needs_post_dma_flush(dev))
		__dma_sync(dma_addr_to_page(dev, dma_addr),
			   dma_addr & ~PAGE_MASK, size, direction);

	plat_unmap_dma_mem(dev, dma_addr, size, direction);
}

static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
	int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
{
	int i;

	for (i = 0; i < nents; i++, sg++) {
		if (!plat_device_is_coherent(dev))
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
#ifdef CONFIG_NEED_SG_DMA_LENGTH
		sg->dma_length = sg->length;
#endif
		sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
				  sg->offset;
	}

	return nents;
}

static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
	unsigned long offset, size_t size, enum dma_data_direction direction,
	struct dma_attrs *attrs)
{
	if (!plat_device_is_coherent(dev))
		__dma_sync(page, offset, size, direction);

	return plat_map_dma_mem_page(dev, page) + offset;
}

static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
	int nhwentries, enum dma_data_direction direction,
	struct dma_attrs *attrs)
{
	int i;

	for (i = 0; i < nhwentries; i++, sg++) {
		if (!plat_device_is_coherent(dev) &&
		    direction != DMA_TO_DEVICE)
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
		plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
	}
}

static void mips_dma_sync_single_for_cpu(struct device *dev,
	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
{
	if (cpu_needs_post_dma_flush(dev))
		__dma_sync(dma_addr_to_page(dev, dma_handle),
			   dma_handle & ~PAGE_MASK, size, direction);
}

static void mips_dma_sync_single_for_device(struct device *dev,
	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
{
	plat_extra_sync_for_device(dev);
	if (!plat_device_is_coherent(dev))
		__dma_sync(dma_addr_to_page(dev, dma_handle),
			   dma_handle & ~PAGE_MASK, size, direction);
}

static void mips_dma_sync_sg_for_cpu(struct device *dev,
	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
{
	int i;

	if (cpu_needs_post_dma_flush(dev))
		for (i = 0; i < nelems; i++, sg++)
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
}

static void mips_dma_sync_sg_for_device(struct device *dev,
	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
{
	int i;

	if (!plat_device_is_coherent(dev))
		for (i = 0; i < nelems; i++, sg++)
			__dma_sync(sg_page(sg), sg->offset, sg->length,
				   direction);
}

int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
{
	return plat_dma_mapping_error(dev, dma_addr);
}

int mips_dma_supported(struct device *dev, u64 mask)
{
	return plat_dma_supported(dev, mask);
}

void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
			 enum dma_data_direction direction)
{
	BUG_ON(direction == DMA_NONE);

	plat_extra_sync_for_device(dev);
	if (!plat_device_is_coherent(dev))
		__dma_sync_virtual(vaddr, size, direction);
}

EXPORT_SYMBOL(dma_cache_sync);

static struct dma_map_ops mips_default_dma_map_ops = {
	.alloc = mips_dma_alloc_coherent,
	.free = mips_dma_free_coherent,
	.map_page = mips_dma_map_page,
	.unmap_page = mips_dma_unmap_page,
	.map_sg = mips_dma_map_sg,
	.unmap_sg = mips_dma_unmap_sg,
	.sync_single_for_cpu = mips_dma_sync_single_for_cpu,
	.sync_single_for_device = mips_dma_sync_single_for_device,
	.sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
	.sync_sg_for_device = mips_dma_sync_sg_for_device,
	.mapping_error = mips_dma_mapping_error,
	.dma_supported = mips_dma_supported
};

struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
EXPORT_SYMBOL(mips_dma_map_ops);

#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)

static int __init mips_dma_init(void)
{
	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);

	return 0;
}
fs_initcall(mips_dma_init);
Commit	Line	Data
1da177e4 LT	1	/*
	2	* This file is subject to the terms and conditions of the GNU General Public
	3	* License. See the file "COPYING" in the main directory of this archive
	4	* for more details.
	5	*
	6	* Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
70342287	7	* Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
1da177e4 LT	8	* swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
1da177e4 LT	9	*/
9a88cbb5	10
1da177e4	11	#include <linux/types.h>
9a88cbb5	12	#include <linux/dma-mapping.h>
1da177e4 LT	13	#include <linux/mm.h>
1da177e4 LT	14	#include <linux/module.h>
4fcc47a0	15	#include <linux/scatterlist.h>
6e86b0bf	16	#include <linux/string.h>
5a0e3ad6	17	#include <linux/gfp.h>
e36863a5	18	#include <linux/highmem.h>
1da177e4 LT	19
1da177e4 LT	20	#include <asm/cache.h>
69f24d17	21	#include <asm/cpu-type.h>
1da177e4 LT	22	#include <asm/io.h>
1da177e4 LT	23
9a88cbb5 RB	24	#include <dma-coherence.h>
9a88cbb5 RB	25
b6d92b4a SH	26	int coherentio = 0; /* User defined DMA coherency from command line. */
	27	EXPORT_SYMBOL_GPL(coherentio);
	28	int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
	29
	30	static int __init setcoherentio(char *str)
	31	{
	32	coherentio = 1;
	33	pr_info("Hardware DMA cache coherency (command line)\n");
	34	return 0;
	35	}
	36	early_param("coherentio", setcoherentio);
	37
	38	static int __init setnocoherentio(char *str)
	39	{
	40	coherentio = 0;
	41	pr_info("Software DMA cache coherency (command line)\n");
	42	return 0;
	43	}
	44	early_param("nocoherentio", setnocoherentio);
	45
e36863a5	46	static inline struct page dma_addr_to_page(struct device dev,
3807ef3f	47	dma_addr_t dma_addr)
c9d06962	48	{
e36863a5 DD	49	return pfn_to_page(
e36863a5 DD	50	plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
c9d06962 FBH	51	}
c9d06962 FBH	52
1da177e4	53	/*
f86f55d3 JQ	54	* The affected CPUs below in 'cpu_needs_post_dma_flush()' can
	55	* speculatively fill random cachelines with stale data at any time,
	56	* requiring an extra flush post-DMA.
	57	*
1da177e4 LT	58	* Warning on the terminology - Linux calls an uncached area coherent;
	59	* MIPS terminology calls memory areas with hardware maintained coherency
	60	* coherent.
	61	*/
f86f55d3	62	static inline int cpu_needs_post_dma_flush(struct device *dev)
9a88cbb5 RB	63	{
9a88cbb5 RB	64	return !plat_device_is_coherent(dev) &&
d451e734	65	(boot_cpu_type() == CPU_R10000 \|\|
eb37e6dd RB	66	boot_cpu_type() == CPU_R12000 \|\|
eb37e6dd RB	67	boot_cpu_type() == CPU_BMIPS5000);
9a88cbb5 RB	68	}
9a88cbb5 RB	69
cce335ae RB	70	static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
cce335ae RB	71	{
a2e715a8 RB	72	gfp_t dma_flag;
a2e715a8 RB	73
cce335ae RB	74	/* ignore region specifiers */
	75	gfp &= ~(__GFP_DMA \| __GFP_DMA32 \| __GFP_HIGHMEM);
	76
a2e715a8	77	#ifdef CONFIG_ISA
cce335ae	78	if (dev == NULL)
a2e715a8	79	dma_flag = __GFP_DMA;
cce335ae RB	80	else
cce335ae RB	81	#endif
a2e715a8	82	#if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
cce335ae	83	if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
a2e715a8 RB	84	dma_flag = __GFP_DMA;
	85	else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	86	dma_flag = __GFP_DMA32;
	87	else
	88	#endif
	89	#if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
	90	if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	91	dma_flag = __GFP_DMA32;
	92	else
	93	#endif
	94	#if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
	95	if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
	96	dma_flag = __GFP_DMA;
cce335ae RB	97	else
cce335ae RB	98	#endif
a2e715a8	99	dma_flag = 0;
cce335ae RB	100
	101	/* Don't invoke OOM killer */
	102	gfp \|= __GFP_NORETRY;
	103
a2e715a8	104	return gfp \| dma_flag;
cce335ae RB	105	}
cce335ae RB	106
1da177e4	107	void dma_alloc_noncoherent(struct device dev, size_t size,
185a8ff5	108	dma_addr_t * dma_handle, gfp_t gfp)
1da177e4 LT	109	{
1da177e4 LT	110	void *ret;
9a88cbb5	111
cce335ae	112	gfp = massage_gfp_flags(dev, gfp);
1da177e4	113
1da177e4 LT	114	ret = (void *) __get_free_pages(gfp, get_order(size));
	115
	116	if (ret != NULL) {
	117	memset(ret, 0, size);
9a88cbb5	118	*dma_handle = plat_map_dma_mem(dev, ret, size);
1da177e4 LT	119	}
	120
	121	return ret;
	122	}
1da177e4 LT	123	EXPORT_SYMBOL(dma_alloc_noncoherent);
1da177e4 LT	124
48e1fd5a	125	static void mips_dma_alloc_coherent(struct device dev, size_t size,
e8d51e54	126	dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
1da177e4 LT	127	{
	128	void *ret;
	129
f8ac0425 YY	130	if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
	131	return ret;
	132
cce335ae	133	gfp = massage_gfp_flags(dev, gfp);
9a88cbb5	134
9a88cbb5 RB	135	ret = (void *) __get_free_pages(gfp, get_order(size));
9a88cbb5 RB	136
1da177e4	137	if (ret) {
9a88cbb5 RB	138	memset(ret, 0, size);
	139	*dma_handle = plat_map_dma_mem(dev, ret, size);
	140
	141	if (!plat_device_is_coherent(dev)) {
	142	dma_cache_wback_inv((unsigned long) ret, size);
b6d92b4a SH	143	if (!hw_coherentio)
b6d92b4a SH	144	ret = UNCAC_ADDR(ret);
9a88cbb5	145	}
1da177e4 LT	146	}
	147
	148	return ret;
	149	}
	150
1da177e4 LT	151
	152	void dma_free_noncoherent(struct device dev, size_t size, void vaddr,
	153	dma_addr_t dma_handle)
	154	{
d3f634b9	155	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
1da177e4 LT	156	free_pages((unsigned long) vaddr, get_order(size));
1da177e4 LT	157	}
1da177e4 LT	158	EXPORT_SYMBOL(dma_free_noncoherent);
1da177e4 LT	159
48e1fd5a	160	static void mips_dma_free_coherent(struct device dev, size_t size, void vaddr,
e8d51e54	161	dma_addr_t dma_handle, struct dma_attrs *attrs)
1da177e4 LT	162	{
1da177e4 LT	163	unsigned long addr = (unsigned long) vaddr;
f8ac0425 YY	164	int order = get_order(size);
	165
	166	if (dma_release_from_coherent(dev, order, vaddr))
	167	return;
1da177e4	168
d3f634b9	169	plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
11531ac2	170
b6d92b4a	171	if (!plat_device_is_coherent(dev) && !hw_coherentio)
9a88cbb5 RB	172	addr = CAC_ADDR(addr);
9a88cbb5 RB	173
1da177e4 LT	174	free_pages(addr, get_order(size));
	175	}
	176
e36863a5	177	static inline void __dma_sync_virtual(void *addr, size_t size,
1da177e4 LT	178	enum dma_data_direction direction)
	179	{
	180	switch (direction) {
	181	case DMA_TO_DEVICE:
e36863a5	182	dma_cache_wback((unsigned long)addr, size);
1da177e4 LT	183	break;
	184
	185	case DMA_FROM_DEVICE:
e36863a5	186	dma_cache_inv((unsigned long)addr, size);
1da177e4 LT	187	break;
	188
	189	case DMA_BIDIRECTIONAL:
e36863a5	190	dma_cache_wback_inv((unsigned long)addr, size);
1da177e4 LT	191	break;
	192
	193	default:
	194	BUG();
	195	}
	196	}
	197
e36863a5 DD	198	/*
	199	* A single sg entry may refer to multiple physically contiguous
	200	* pages. But we still need to process highmem pages individually.
	201	* If highmem is not configured then the bulk of this loop gets
	202	* optimized out.
	203	*/
	204	static inline void __dma_sync(struct page *page,
	205	unsigned long offset, size_t size, enum dma_data_direction direction)
	206	{
	207	size_t left = size;
	208
	209	do {
	210	size_t len = left;
	211
	212	if (PageHighMem(page)) {
	213	void *addr;
	214
	215	if (offset + len > PAGE_SIZE) {
	216	if (offset >= PAGE_SIZE) {
	217	page += offset >> PAGE_SHIFT;
	218	offset &= ~PAGE_MASK;
	219	}
	220	len = PAGE_SIZE - offset;
	221	}
	222
	223	addr = kmap_atomic(page);
	224	__dma_sync_virtual(addr + offset, len, direction);
	225	kunmap_atomic(addr);
	226	} else
	227	__dma_sync_virtual(page_address(page) + offset,
	228	size, direction);
	229	offset = 0;
	230	page++;
	231	left -= len;
	232	} while (left);
	233	}
	234
48e1fd5a DD	235	static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
48e1fd5a DD	236	size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
1da177e4	237	{
f86f55d3	238	if (cpu_needs_post_dma_flush(dev))
e36863a5 DD	239	__dma_sync(dma_addr_to_page(dev, dma_addr),
e36863a5 DD	240	dma_addr & ~PAGE_MASK, size, direction);
1da177e4	241
d3f634b9	242	plat_unmap_dma_mem(dev, dma_addr, size, direction);
1da177e4 LT	243	}
1da177e4 LT	244
48e1fd5a DD	245	static int mips_dma_map_sg(struct device dev, struct scatterlist sg,
48e1fd5a DD	246	int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
1da177e4 LT	247	{
	248	int i;
	249
1da177e4	250	for (i = 0; i < nents; i++, sg++) {
e36863a5 DD	251	if (!plat_device_is_coherent(dev))
	252	__dma_sync(sg_page(sg), sg->offset, sg->length,
	253	direction);
4954a9a2 J	254	#ifdef CONFIG_NEED_SG_DMA_LENGTH
	255	sg->dma_length = sg->length;
	256	#endif
e36863a5 DD	257	sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
e36863a5 DD	258	sg->offset;
1da177e4 LT	259	}
	260
	261	return nents;
	262	}
	263
48e1fd5a DD	264	static dma_addr_t mips_dma_map_page(struct device dev, struct page page,
	265	unsigned long offset, size_t size, enum dma_data_direction direction,
	266	struct dma_attrs *attrs)
1da177e4	267	{
48e1fd5a	268	if (!plat_device_is_coherent(dev))
e36863a5	269	__dma_sync(page, offset, size, direction);
1da177e4	270
e36863a5	271	return plat_map_dma_mem_page(dev, page) + offset;
1da177e4 LT	272	}
1da177e4 LT	273
48e1fd5a DD	274	static void mips_dma_unmap_sg(struct device dev, struct scatterlist sg,
	275	int nhwentries, enum dma_data_direction direction,
	276	struct dma_attrs *attrs)
1da177e4	277	{
1da177e4 LT	278	int i;
1da177e4 LT	279
1da177e4	280	for (i = 0; i < nhwentries; i++, sg++) {
9a88cbb5	281	if (!plat_device_is_coherent(dev) &&
e36863a5 DD	282	direction != DMA_TO_DEVICE)
	283	__dma_sync(sg_page(sg), sg->offset, sg->length,
	284	direction);
d3f634b9	285	plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
1da177e4 LT	286	}
	287	}
	288
48e1fd5a DD	289	static void mips_dma_sync_single_for_cpu(struct device *dev,
48e1fd5a DD	290	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
1da177e4	291	{
f86f55d3	292	if (cpu_needs_post_dma_flush(dev))
e36863a5 DD	293	__dma_sync(dma_addr_to_page(dev, dma_handle),
e36863a5 DD	294	dma_handle & ~PAGE_MASK, size, direction);
1da177e4 LT	295	}
1da177e4 LT	296
48e1fd5a DD	297	static void mips_dma_sync_single_for_device(struct device *dev,
48e1fd5a DD	298	dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
1da177e4	299	{
843aef49	300	plat_extra_sync_for_device(dev);
e36863a5 DD	301	if (!plat_device_is_coherent(dev))
	302	__dma_sync(dma_addr_to_page(dev, dma_handle),
	303	dma_handle & ~PAGE_MASK, size, direction);
1da177e4 LT	304	}
1da177e4 LT	305
48e1fd5a DD	306	static void mips_dma_sync_sg_for_cpu(struct device *dev,
48e1fd5a DD	307	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
1da177e4 LT	308	{
1da177e4 LT	309	int i;
42a3b4f2	310
55c25c2f J	311	if (cpu_needs_post_dma_flush(dev))
55c25c2f J	312	for (i = 0; i < nelems; i++, sg++)
e36863a5 DD	313	__dma_sync(sg_page(sg), sg->offset, sg->length,
e36863a5 DD	314	direction);
1da177e4 LT	315	}
1da177e4 LT	316
48e1fd5a DD	317	static void mips_dma_sync_sg_for_device(struct device *dev,
48e1fd5a DD	318	struct scatterlist *sg, int nelems, enum dma_data_direction direction)
1da177e4 LT	319	{
	320	int i;
	321
55c25c2f J	322	if (!plat_device_is_coherent(dev))
55c25c2f J	323	for (i = 0; i < nelems; i++, sg++)
e36863a5 DD	324	__dma_sync(sg_page(sg), sg->offset, sg->length,
e36863a5 DD	325	direction);
1da177e4 LT	326	}
1da177e4 LT	327
48e1fd5a	328	int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
1da177e4	329	{
843aef49	330	return plat_dma_mapping_error(dev, dma_addr);
1da177e4 LT	331	}
1da177e4 LT	332
48e1fd5a	333	int mips_dma_supported(struct device *dev, u64 mask)
1da177e4	334	{
843aef49	335	return plat_dma_supported(dev, mask);
1da177e4 LT	336	}
1da177e4 LT	337
a3aad4aa	338	void dma_cache_sync(struct device dev, void vaddr, size_t size,
48e1fd5a	339	enum dma_data_direction direction)
1da177e4	340	{
9a88cbb5	341	BUG_ON(direction == DMA_NONE);
1da177e4	342
843aef49	343	plat_extra_sync_for_device(dev);
9a88cbb5	344	if (!plat_device_is_coherent(dev))
e36863a5	345	__dma_sync_virtual(vaddr, size, direction);
1da177e4 LT	346	}
1da177e4 LT	347
a3aad4aa RB	348	EXPORT_SYMBOL(dma_cache_sync);
a3aad4aa RB	349
48e1fd5a	350	static struct dma_map_ops mips_default_dma_map_ops = {
e8d51e54 AP	351	.alloc = mips_dma_alloc_coherent,
e8d51e54 AP	352	.free = mips_dma_free_coherent,
48e1fd5a DD	353	.map_page = mips_dma_map_page,
	354	.unmap_page = mips_dma_unmap_page,
	355	.map_sg = mips_dma_map_sg,
	356	.unmap_sg = mips_dma_unmap_sg,
	357	.sync_single_for_cpu = mips_dma_sync_single_for_cpu,
	358	.sync_single_for_device = mips_dma_sync_single_for_device,
	359	.sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
	360	.sync_sg_for_device = mips_dma_sync_sg_for_device,
	361	.mapping_error = mips_dma_mapping_error,
	362	.dma_supported = mips_dma_supported
	363	};
	364
	365	struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
	366	EXPORT_SYMBOL(mips_dma_map_ops);
	367
	368	#define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
	369
	370	static int __init mips_dma_init(void)
	371	{
	372	dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
	373
	374	return 0;
	375	}
	376	fs_initcall(mips_dma_init);