[mirror_ubuntu-zesty-kernel.git] / arch / sparc / mm / iommu.c

/*
 * iommu.c:  IOMMU specific routines for memory management.
 *
 * Copyright (C) 1995 David S. Miller  (davem@caip.rutgers.edu)
 * Copyright (C) 1995,2002 Pete Zaitcev     (zaitcev@yahoo.com)
 * Copyright (C) 1996 Eddie C. Dost    (ecd@skynet.be)
 * Copyright (C) 1997,1998 Jakub Jelinek    (jj@sunsite.mff.cuni.cz)
 */
 
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/highmem.h>	/* pte_offset_map => kmap_atomic */
#include <linux/scatterlist.h>
#include <linux/of.h>
#include <linux/of_device.h>

#include <asm/pgalloc.h>
#include <asm/pgtable.h>
#include <asm/io.h>
#include <asm/mxcc.h>
#include <asm/mbus.h>
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
#include <asm/bitext.h>
#include <asm/iommu.h>
#include <asm/dma.h>

/*
 * This can be sized dynamically, but we will do this
 * only when we have a guidance about actual I/O pressures.
 */
#define IOMMU_RNGE	IOMMU_RNGE_256MB
#define IOMMU_START	0xF0000000
#define IOMMU_WINSIZE	(256*1024*1024U)
#define IOMMU_NPTES	(IOMMU_WINSIZE/PAGE_SIZE)	/* 64K PTEs, 265KB */
#define IOMMU_ORDER	6				/* 4096 * (1<<6) */

/* srmmu.c */
extern int viking_mxcc_present;
BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)
extern int flush_page_for_dma_global;
static int viking_flush;
/* viking.S */
extern void viking_flush_page(unsigned long page);
extern void viking_mxcc_flush_page(unsigned long page);

/*
 * Values precomputed according to CPU type.
 */
static unsigned int ioperm_noc;		/* Consistent mapping iopte flags */
static pgprot_t dvma_prot;		/* Consistent mapping pte flags */

#define IOPERM        (IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID)
#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) | (perm)) & ~IOPTE_WAZ)

static void __init sbus_iommu_init(struct platform_device *op)
{
	struct iommu_struct *iommu;
	unsigned int impl, vers;
	unsigned long *bitmap;
	unsigned long tmp;

	iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
	if (!iommu) {
		prom_printf("Unable to allocate iommu structure\n");
		prom_halt();
	}

	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
				 "iommu_regs");
	if (!iommu->regs) {
		prom_printf("Cannot map IOMMU registers\n");
		prom_halt();
	}
	impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28;
	vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24;
	tmp = iommu->regs->control;
	tmp &= ~(IOMMU_CTRL_RNGE);
	tmp |= (IOMMU_RNGE_256MB | IOMMU_CTRL_ENAB);
	iommu->regs->control = tmp;
	iommu_invalidate(iommu->regs);
	iommu->start = IOMMU_START;
	iommu->end = 0xffffffff;

	/* Allocate IOMMU page table */
	/* Stupid alignment constraints give me a headache. 
	   We need 256K or 512K or 1M or 2M area aligned to
           its size and current gfp will fortunately give
           it to us. */
        tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
	if (!tmp) {
		prom_printf("Unable to allocate iommu table [0x%08x]\n",
			    IOMMU_NPTES*sizeof(iopte_t));
		prom_halt();
	}
	iommu->page_table = (iopte_t *)tmp;

	/* Initialize new table. */
	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
	flush_cache_all();
	flush_tlb_all();
	iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4;
	iommu_invalidate(iommu->regs);

	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
	if (!bitmap) {
		prom_printf("Unable to allocate iommu bitmap [%d]\n",
			    (int)(IOMMU_NPTES>>3));
		prom_halt();
	}
	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
	/* To be coherent on HyperSparc, the page color of DVMA
	 * and physical addresses must match.
	 */
	if (srmmu_modtype == HyperSparc)
		iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
	else
		iommu->usemap.num_colors = 1;

	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
	       impl, vers, iommu->page_table,
	       (int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);

	op->dev.archdata.iommu = iommu;
}

static int __init iommu_init(void)
{
	struct device_node *dp;

	for_each_node_by_name(dp, "iommu") {
		struct platform_device *op = of_find_device_by_node(dp);

		sbus_iommu_init(op);
		of_propagate_archdata(op);
	}

	return 0;
}

subsys_initcall(iommu_init);

/* This begs to be btfixup-ed by srmmu. */
/* Flush the iotlb entries to ram. */
/* This could be better if we didn't have to flush whole pages. */
static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
{
	unsigned long start;
	unsigned long end;

	start = (unsigned long)iopte;
	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
	start &= PAGE_MASK;
	if (viking_mxcc_present) {
		while(start < end) {
			viking_mxcc_flush_page(start);
			start += PAGE_SIZE;
		}
	} else if (viking_flush) {
		while(start < end) {
			viking_flush_page(start);
			start += PAGE_SIZE;
		}
	} else {
		while(start < end) {
			__flush_page_to_ram(start);
			start += PAGE_SIZE;
		}
	}
}

static u32 iommu_get_one(struct device *dev, struct page *page, int npages)
{
	struct iommu_struct *iommu = dev->archdata.iommu;
	int ioptex;
	iopte_t *iopte, *iopte0;
	unsigned int busa, busa0;
	int i;

	/* page color = pfn of page */
	ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page));
	if (ioptex < 0)
		panic("iommu out");
	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
	iopte0 = &iommu->page_table[ioptex];

	busa = busa0;
	iopte = iopte0;
	for (i = 0; i < npages; i++) {
		iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM);
		iommu_invalidate_page(iommu->regs, busa);
		busa += PAGE_SIZE;
		iopte++;
		page++;
	}

	iommu_flush_iotlb(iopte0, npages);

	return busa0;
}

static u32 iommu_get_scsi_one(struct device *dev, char *vaddr, unsigned int len)
{
	unsigned long off;
	int npages;
	struct page *page;
	u32 busa;

	off = (unsigned long)vaddr & ~PAGE_MASK;
	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	page = virt_to_page((unsigned long)vaddr & PAGE_MASK);
	busa = iommu_get_one(dev, page, npages);
	return busa + off;
}

static __u32 iommu_get_scsi_one_noflush(struct device *dev, char *vaddr, unsigned long len)
{
	return iommu_get_scsi_one(dev, vaddr, len);
}

static __u32 iommu_get_scsi_one_gflush(struct device *dev, char *vaddr, unsigned long len)
{
	flush_page_for_dma(0);
	return iommu_get_scsi_one(dev, vaddr, len);
}

static __u32 iommu_get_scsi_one_pflush(struct device *dev, char *vaddr, unsigned long len)
{
	unsigned long page = ((unsigned long) vaddr) & PAGE_MASK;

	while(page < ((unsigned long)(vaddr + len))) {
		flush_page_for_dma(page);
		page += PAGE_SIZE;
	}
	return iommu_get_scsi_one(dev, vaddr, len);
}

static void iommu_get_scsi_sgl_noflush(struct device *dev, struct scatterlist *sg, int sz)
{
	int n;

	while (sz != 0) {
		--sz;
		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
		sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
		sg->dma_length = sg->length;
		sg = sg_next(sg);
	}
}

static void iommu_get_scsi_sgl_gflush(struct device *dev, struct scatterlist *sg, int sz)
{
	int n;

	flush_page_for_dma(0);
	while (sz != 0) {
		--sz;
		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
		sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
		sg->dma_length = sg->length;
		sg = sg_next(sg);
	}
}

static void iommu_get_scsi_sgl_pflush(struct device *dev, struct scatterlist *sg, int sz)
{
	unsigned long page, oldpage = 0;
	int n, i;

	while(sz != 0) {
		--sz;

		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;

		/*
		 * We expect unmapped highmem pages to be not in the cache.
		 * XXX Is this a good assumption?
		 * XXX What if someone else unmaps it here and races us?
		 */
		if ((page = (unsigned long) page_address(sg_page(sg))) != 0) {
			for (i = 0; i < n; i++) {
				if (page != oldpage) {	/* Already flushed? */
					flush_page_for_dma(page);
					oldpage = page;
				}
				page += PAGE_SIZE;
			}
		}

		sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
		sg->dma_length = sg->length;
		sg = sg_next(sg);
	}
}

static void iommu_release_one(struct device *dev, u32 busa, int npages)
{
	struct iommu_struct *iommu = dev->archdata.iommu;
	int ioptex;
	int i;

	BUG_ON(busa < iommu->start);
	ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	for (i = 0; i < npages; i++) {
		iopte_val(iommu->page_table[ioptex + i]) = 0;
		iommu_invalidate_page(iommu->regs, busa);
		busa += PAGE_SIZE;
	}
	bit_map_clear(&iommu->usemap, ioptex, npages);
}

static void iommu_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len)
{
	unsigned long off;
	int npages;

	off = vaddr & ~PAGE_MASK;
	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	iommu_release_one(dev, vaddr & PAGE_MASK, npages);
}

static void iommu_release_scsi_sgl(struct device *dev, struct scatterlist *sg, int sz)
{
	int n;

	while(sz != 0) {
		--sz;

		n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
		iommu_release_one(dev, sg->dma_address & PAGE_MASK, n);
		sg->dma_address = 0x21212121;
		sg = sg_next(sg);
	}
}

#ifdef CONFIG_SBUS
static int iommu_map_dma_area(struct device *dev, dma_addr_t *pba, unsigned long va,
			      unsigned long addr, int len)
{
	struct iommu_struct *iommu = dev->archdata.iommu;
	unsigned long page, end;
	iopte_t *iopte = iommu->page_table;
	iopte_t *first;
	int ioptex;

	BUG_ON((va & ~PAGE_MASK) != 0);
	BUG_ON((addr & ~PAGE_MASK) != 0);
	BUG_ON((len & ~PAGE_MASK) != 0);

	/* page color = physical address */
	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
		addr >> PAGE_SHIFT);
	if (ioptex < 0)
		panic("iommu out");

	iopte += ioptex;
	first = iopte;
	end = addr + len;
	while(addr < end) {
		page = va;
		{
			pgd_t *pgdp;
			pmd_t *pmdp;
			pte_t *ptep;

			if (viking_mxcc_present)
				viking_mxcc_flush_page(page);
			else if (viking_flush)
				viking_flush_page(page);
			else
				__flush_page_to_ram(page);

			pgdp = pgd_offset(&init_mm, addr);
			pmdp = pmd_offset(pgdp, addr);
			ptep = pte_offset_map(pmdp, addr);

			set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
		}
		iopte_val(*iopte++) =
		    MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
		addr += PAGE_SIZE;
		va += PAGE_SIZE;
	}
	/* P3: why do we need this?
	 *
	 * DAVEM: Because there are several aspects, none of which
	 *        are handled by a single interface.  Some cpus are
	 *        completely not I/O DMA coherent, and some have
	 *        virtually indexed caches.  The driver DMA flushing
	 *        methods handle the former case, but here during
	 *        IOMMU page table modifications, and usage of non-cacheable
	 *        cpu mappings of pages potentially in the cpu caches, we have
	 *        to handle the latter case as well.
	 */
	flush_cache_all();
	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
	flush_tlb_all();
	iommu_invalidate(iommu->regs);

	*pba = iommu->start + (ioptex << PAGE_SHIFT);
	return 0;
}

static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len)
{
	struct iommu_struct *iommu = dev->archdata.iommu;
	iopte_t *iopte = iommu->page_table;
	unsigned long end;
	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;

	BUG_ON((busa & ~PAGE_MASK) != 0);
	BUG_ON((len & ~PAGE_MASK) != 0);

	iopte += ioptex;
	end = busa + len;
	while (busa < end) {
		iopte_val(*iopte++) = 0;
		busa += PAGE_SIZE;
	}
	flush_tlb_all();
	iommu_invalidate(iommu->regs);
	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
}
#endif

static const struct sparc32_dma_ops iommu_dma_noflush_ops = {
	.get_scsi_one		= iommu_get_scsi_one_noflush,
	.get_scsi_sgl		= iommu_get_scsi_sgl_noflush,
	.release_scsi_one	= iommu_release_scsi_one,
	.release_scsi_sgl	= iommu_release_scsi_sgl,
#ifdef CONFIG_SBUS
	.map_dma_area		= iommu_map_dma_area,
	.unmap_dma_area		= iommu_unmap_dma_area,
#endif
};

static const struct sparc32_dma_ops iommu_dma_gflush_ops = {
	.get_scsi_one		= iommu_get_scsi_one_gflush,
	.get_scsi_sgl		= iommu_get_scsi_sgl_gflush,
	.release_scsi_one	= iommu_release_scsi_one,
	.release_scsi_sgl	= iommu_release_scsi_sgl,
#ifdef CONFIG_SBUS
	.map_dma_area		= iommu_map_dma_area,
	.unmap_dma_area		= iommu_unmap_dma_area,
#endif
};

static const struct sparc32_dma_ops iommu_dma_pflush_ops = {
	.get_scsi_one		= iommu_get_scsi_one_pflush,
	.get_scsi_sgl		= iommu_get_scsi_sgl_pflush,
	.release_scsi_one	= iommu_release_scsi_one,
	.release_scsi_sgl	= iommu_release_scsi_sgl,
#ifdef CONFIG_SBUS
	.map_dma_area		= iommu_map_dma_area,
	.unmap_dma_area		= iommu_unmap_dma_area,
#endif
};

void __init ld_mmu_iommu(void)
{
	viking_flush = (BTFIXUPVAL_CALL(flush_page_for_dma) == (unsigned long)viking_flush_page);

	if (!BTFIXUPVAL_CALL(flush_page_for_dma)) {
		/* IO coherent chip */
		sparc32_dma_ops = &iommu_dma_noflush_ops;
	} else if (flush_page_for_dma_global) {
		/* flush_page_for_dma flushes everything, no matter of what page is it */
		sparc32_dma_ops = &iommu_dma_gflush_ops;
	} else {
		sparc32_dma_ops = &iommu_dma_pflush_ops;
	}

	if (viking_mxcc_present || srmmu_modtype == HyperSparc) {
		dvma_prot = __pgprot(SRMMU_CACHE | SRMMU_ET_PTE | SRMMU_PRIV);
		ioperm_noc = IOPTE_CACHE | IOPTE_WRITE | IOPTE_VALID;
	} else {
		dvma_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV);
		ioperm_noc = IOPTE_WRITE | IOPTE_VALID;
	}
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* iommu.c: IOMMU specific routines for memory management.
	3	*
	4	* Copyright (C) 1995 David S. Miller (davem@caip.rutgers.edu)
	5	* Copyright (C) 1995,2002 Pete Zaitcev (zaitcev@yahoo.com)
	6	* Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be)
	7	* Copyright (C) 1997,1998 Jakub Jelinek (jj@sunsite.mff.cuni.cz)
	8	*/
	9
1da177e4 LT	10	#include <linux/kernel.h>
	11	#include <linux/init.h>
	12	#include <linux/mm.h>
	13	#include <linux/slab.h>
	14	#include <linux/highmem.h> /* pte_offset_map => kmap_atomic */
0912a5db	15	#include <linux/scatterlist.h>
9dc69230 DM	16	#include <linux/of.h>
9dc69230 DM	17	#include <linux/of_device.h>
1da177e4	18
1da177e4 LT	19	#include <asm/pgalloc.h>
1da177e4 LT	20	#include <asm/pgtable.h>
1da177e4 LT	21	#include <asm/io.h>
	22	#include <asm/mxcc.h>
	23	#include <asm/mbus.h>
	24	#include <asm/cacheflush.h>
	25	#include <asm/tlbflush.h>
	26	#include <asm/bitext.h>
	27	#include <asm/iommu.h>
	28	#include <asm/dma.h>
	29
	30	/*
	31	* This can be sized dynamically, but we will do this
	32	* only when we have a guidance about actual I/O pressures.
	33	*/
	34	#define IOMMU_RNGE IOMMU_RNGE_256MB
	35	#define IOMMU_START 0xF0000000
	36	#define IOMMU_WINSIZE (25610241024U)
	37	#define IOMMU_NPTES (IOMMU_WINSIZE/PAGE_SIZE) /* 64K PTEs, 265KB */
	38	#define IOMMU_ORDER 6 /* 4096 * (1<<6) */
	39
	40	/* srmmu.c */
	41	extern int viking_mxcc_present;
	42	BTFIXUPDEF_CALL(void, flush_page_for_dma, unsigned long)
	43	#define flush_page_for_dma(page) BTFIXUP_CALL(flush_page_for_dma)(page)
	44	extern int flush_page_for_dma_global;
	45	static int viking_flush;
	46	/* viking.S */
	47	extern void viking_flush_page(unsigned long page);
	48	extern void viking_mxcc_flush_page(unsigned long page);
	49
	50	/*
	51	* Values precomputed according to CPU type.
	52	*/
	53	static unsigned int ioperm_noc; /* Consistent mapping iopte flags */
	54	static pgprot_t dvma_prot; /* Consistent mapping pte flags */
	55
	56	#define IOPERM (IOPTE_CACHE \| IOPTE_WRITE \| IOPTE_VALID)
	57	#define MKIOPTE(pfn, perm) (((((pfn)<<8) & IOPTE_PAGE) \| (perm)) & ~IOPTE_WAZ)
	58
cd4cd730	59	static void __init sbus_iommu_init(struct platform_device *op)
1da177e4	60	{
1da177e4	61	struct iommu_struct *iommu;
e0039348	62	unsigned int impl, vers;
1da177e4	63	unsigned long *bitmap;
e0039348 DM	64	unsigned long tmp;
e0039348 DM	65
71cd03b0	66	iommu = kmalloc(sizeof(struct iommu_struct), GFP_KERNEL);
1da177e4 LT	67	if (!iommu) {
	68	prom_printf("Unable to allocate iommu structure\n");
	69	prom_halt();
	70	}
e0039348	71
046e26a8	72	iommu->regs = of_ioremap(&op->resource[0], 0, PAGE_SIZE * 3,
e0039348	73	"iommu_regs");
1da177e4 LT	74	if (!iommu->regs) {
	75	prom_printf("Cannot map IOMMU registers\n");
	76	prom_halt();
	77	}
	78	impl = (iommu->regs->control & IOMMU_CTRL_IMPL) >> 28;
	79	vers = (iommu->regs->control & IOMMU_CTRL_VERS) >> 24;
	80	tmp = iommu->regs->control;
	81	tmp &= ~(IOMMU_CTRL_RNGE);
	82	tmp \|= (IOMMU_RNGE_256MB \| IOMMU_CTRL_ENAB);
	83	iommu->regs->control = tmp;
	84	iommu_invalidate(iommu->regs);
	85	iommu->start = IOMMU_START;
	86	iommu->end = 0xffffffff;
	87
	88	/* Allocate IOMMU page table */
	89	/* Stupid alignment constraints give me a headache.
	90	We need 256K or 512K or 1M or 2M area aligned to
	91	its size and current gfp will fortunately give
	92	it to us. */
	93	tmp = __get_free_pages(GFP_KERNEL, IOMMU_ORDER);
	94	if (!tmp) {
	95	prom_printf("Unable to allocate iommu table [0x%08x]\n",
	96	IOMMU_NPTES*sizeof(iopte_t));
	97	prom_halt();
	98	}
	99	iommu->page_table = (iopte_t *)tmp;
	100
	101	/* Initialize new table. */
	102	memset(iommu->page_table, 0, IOMMU_NPTES*sizeof(iopte_t));
	103	flush_cache_all();
	104	flush_tlb_all();
	105	iommu->regs->base = __pa((unsigned long) iommu->page_table) >> 4;
	106	iommu_invalidate(iommu->regs);
	107
	108	bitmap = kmalloc(IOMMU_NPTES>>3, GFP_KERNEL);
	109	if (!bitmap) {
	110	prom_printf("Unable to allocate iommu bitmap [%d]\n",
	111	(int)(IOMMU_NPTES>>3));
	112	prom_halt();
	113	}
	114	bit_map_init(&iommu->usemap, bitmap, IOMMU_NPTES);
	115	/* To be coherent on HyperSparc, the page color of DVMA
	116	* and physical addresses must match.
	117	*/
	118	if (srmmu_modtype == HyperSparc)
	119	iommu->usemap.num_colors = vac_cache_size >> PAGE_SHIFT;
	120	else
	121	iommu->usemap.num_colors = 1;
	122
046e26a8 DM	123	printk(KERN_INFO "IOMMU: impl %d vers %d table 0x%p[%d B] map [%d b]\n",
	124	impl, vers, iommu->page_table,
	125	(int)(IOMMU_NPTES*sizeof(iopte_t)), (int)IOMMU_NPTES);
1da177e4	126
e0039348	127	op->dev.archdata.iommu = iommu;
1da177e4 LT	128	}
1da177e4 LT	129
046e26a8 DM	130	static int __init iommu_init(void)
	131	{
	132	struct device_node *dp;
	133
	134	for_each_node_by_name(dp, "iommu") {
cd4cd730	135	struct platform_device *op = of_find_device_by_node(dp);
046e26a8 DM	136
	137	sbus_iommu_init(op);
	138	of_propagate_archdata(op);
	139	}
	140
	141	return 0;
	142	}
	143
	144	subsys_initcall(iommu_init);
	145
1da177e4 LT	146	/* This begs to be btfixup-ed by srmmu. */
	147	/* Flush the iotlb entries to ram. */
	148	/* This could be better if we didn't have to flush whole pages. */
	149	static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte)
	150	{
	151	unsigned long start;
	152	unsigned long end;
	153
3185d4d2	154	start = (unsigned long)iopte;
1da177e4	155	end = PAGE_ALIGN(start + niopte*sizeof(iopte_t));
3185d4d2	156	start &= PAGE_MASK;
1da177e4 LT	157	if (viking_mxcc_present) {
	158	while(start < end) {
	159	viking_mxcc_flush_page(start);
	160	start += PAGE_SIZE;
	161	}
	162	} else if (viking_flush) {
	163	while(start < end) {
	164	viking_flush_page(start);
	165	start += PAGE_SIZE;
	166	}
	167	} else {
	168	while(start < end) {
	169	__flush_page_to_ram(start);
	170	start += PAGE_SIZE;
	171	}
	172	}
	173	}
	174
260489fa	175	static u32 iommu_get_one(struct device dev, struct page page, int npages)
1da177e4	176	{
260489fa	177	struct iommu_struct *iommu = dev->archdata.iommu;
1da177e4 LT	178	int ioptex;
	179	iopte_t iopte, iopte0;
	180	unsigned int busa, busa0;
	181	int i;
	182
	183	/* page color = pfn of page */
	184	ioptex = bit_map_string_get(&iommu->usemap, npages, page_to_pfn(page));
	185	if (ioptex < 0)
	186	panic("iommu out");
	187	busa0 = iommu->start + (ioptex << PAGE_SHIFT);
	188	iopte0 = &iommu->page_table[ioptex];
	189
	190	busa = busa0;
	191	iopte = iopte0;
	192	for (i = 0; i < npages; i++) {
	193	iopte_val(*iopte) = MKIOPTE(page_to_pfn(page), IOPERM);
	194	iommu_invalidate_page(iommu->regs, busa);
	195	busa += PAGE_SIZE;
	196	iopte++;
	197	page++;
	198	}
	199
	200	iommu_flush_iotlb(iopte0, npages);
	201
	202	return busa0;
	203	}
	204
260489fa	205	static u32 iommu_get_scsi_one(struct device dev, char vaddr, unsigned int len)
1da177e4 LT	206	{
	207	unsigned long off;
	208	int npages;
	209	struct page *page;
	210	u32 busa;
	211
	212	off = (unsigned long)vaddr & ~PAGE_MASK;
	213	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
	214	page = virt_to_page((unsigned long)vaddr & PAGE_MASK);
260489fa	215	busa = iommu_get_one(dev, page, npages);
1da177e4 LT	216	return busa + off;
	217	}
	218
260489fa	219	static __u32 iommu_get_scsi_one_noflush(struct device dev, char vaddr, unsigned long len)
1da177e4	220	{
260489fa	221	return iommu_get_scsi_one(dev, vaddr, len);
1da177e4 LT	222	}
1da177e4 LT	223
260489fa	224	static __u32 iommu_get_scsi_one_gflush(struct device dev, char vaddr, unsigned long len)
1da177e4 LT	225	{
1da177e4 LT	226	flush_page_for_dma(0);
260489fa	227	return iommu_get_scsi_one(dev, vaddr, len);
1da177e4 LT	228	}
1da177e4 LT	229
260489fa	230	static __u32 iommu_get_scsi_one_pflush(struct device dev, char vaddr, unsigned long len)
1da177e4 LT	231	{
	232	unsigned long page = ((unsigned long) vaddr) & PAGE_MASK;
	233
	234	while(page < ((unsigned long)(vaddr + len))) {
	235	flush_page_for_dma(page);
	236	page += PAGE_SIZE;
	237	}
260489fa	238	return iommu_get_scsi_one(dev, vaddr, len);
1da177e4 LT	239	}
1da177e4 LT	240
260489fa	241	static void iommu_get_scsi_sgl_noflush(struct device dev, struct scatterlist sg, int sz)
1da177e4 LT	242	{
	243	int n;
	244
	245	while (sz != 0) {
	246	--sz;
	247	n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
aa83a26a RR	248	sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
aa83a26a RR	249	sg->dma_length = sg->length;
0912a5db	250	sg = sg_next(sg);
1da177e4 LT	251	}
	252	}
	253
260489fa	254	static void iommu_get_scsi_sgl_gflush(struct device dev, struct scatterlist sg, int sz)
1da177e4 LT	255	{
	256	int n;
	257
	258	flush_page_for_dma(0);
	259	while (sz != 0) {
	260	--sz;
	261	n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
aa83a26a RR	262	sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
aa83a26a RR	263	sg->dma_length = sg->length;
0912a5db	264	sg = sg_next(sg);
1da177e4 LT	265	}
	266	}
	267
260489fa	268	static void iommu_get_scsi_sgl_pflush(struct device dev, struct scatterlist sg, int sz)
1da177e4 LT	269	{
	270	unsigned long page, oldpage = 0;
	271	int n, i;
	272
	273	while(sz != 0) {
	274	--sz;
	275
	276	n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
	277
	278	/*
	279	* We expect unmapped highmem pages to be not in the cache.
	280	* XXX Is this a good assumption?
	281	* XXX What if someone else unmaps it here and races us?
	282	*/
58b053e4	283	if ((page = (unsigned long) page_address(sg_page(sg))) != 0) {
1da177e4 LT	284	for (i = 0; i < n; i++) {
	285	if (page != oldpage) { /* Already flushed? */
	286	flush_page_for_dma(page);
	287	oldpage = page;
	288	}
	289	page += PAGE_SIZE;
	290	}
	291	}
	292
aa83a26a RR	293	sg->dma_address = iommu_get_one(dev, sg_page(sg), n) + sg->offset;
aa83a26a RR	294	sg->dma_length = sg->length;
0912a5db	295	sg = sg_next(sg);
1da177e4 LT	296	}
	297	}
	298
260489fa	299	static void iommu_release_one(struct device *dev, u32 busa, int npages)
1da177e4	300	{
260489fa	301	struct iommu_struct *iommu = dev->archdata.iommu;
1da177e4 LT	302	int ioptex;
	303	int i;
	304
1ae61388	305	BUG_ON(busa < iommu->start);
1da177e4 LT	306	ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	307	for (i = 0; i < npages; i++) {
	308	iopte_val(iommu->page_table[ioptex + i]) = 0;
	309	iommu_invalidate_page(iommu->regs, busa);
	310	busa += PAGE_SIZE;
	311	}
	312	bit_map_clear(&iommu->usemap, ioptex, npages);
	313	}
	314
260489fa	315	static void iommu_release_scsi_one(struct device *dev, __u32 vaddr, unsigned long len)
1da177e4 LT	316	{
	317	unsigned long off;
	318	int npages;
	319
	320	off = vaddr & ~PAGE_MASK;
	321	npages = (off + len + PAGE_SIZE-1) >> PAGE_SHIFT;
260489fa	322	iommu_release_one(dev, vaddr & PAGE_MASK, npages);
1da177e4 LT	323	}
1da177e4 LT	324
260489fa	325	static void iommu_release_scsi_sgl(struct device dev, struct scatterlist sg, int sz)
1da177e4 LT	326	{
	327	int n;
	328
	329	while(sz != 0) {
	330	--sz;
	331
	332	n = (sg->length + sg->offset + PAGE_SIZE-1) >> PAGE_SHIFT;
aa83a26a RR	333	iommu_release_one(dev, sg->dma_address & PAGE_MASK, n);
aa83a26a RR	334	sg->dma_address = 0x21212121;
0912a5db	335	sg = sg_next(sg);
1da177e4 LT	336	}
	337	}
	338
	339	#ifdef CONFIG_SBUS
4b1c5df2 DM	340	static int iommu_map_dma_area(struct device dev, dma_addr_t pba, unsigned long va,
4b1c5df2 DM	341	unsigned long addr, int len)
1da177e4	342	{
4b1c5df2	343	struct iommu_struct *iommu = dev->archdata.iommu;
1da177e4	344	unsigned long page, end;
1da177e4 LT	345	iopte_t *iopte = iommu->page_table;
	346	iopte_t *first;
	347	int ioptex;
	348
1ae61388 ES	349	BUG_ON((va & ~PAGE_MASK) != 0);
	350	BUG_ON((addr & ~PAGE_MASK) != 0);
	351	BUG_ON((len & ~PAGE_MASK) != 0);
1da177e4 LT	352
	353	/* page color = physical address */
	354	ioptex = bit_map_string_get(&iommu->usemap, len >> PAGE_SHIFT,
	355	addr >> PAGE_SHIFT);
	356	if (ioptex < 0)
	357	panic("iommu out");
	358
	359	iopte += ioptex;
	360	first = iopte;
	361	end = addr + len;
	362	while(addr < end) {
	363	page = va;
	364	{
	365	pgd_t *pgdp;
	366	pmd_t *pmdp;
	367	pte_t *ptep;
	368
	369	if (viking_mxcc_present)
	370	viking_mxcc_flush_page(page);
	371	else if (viking_flush)
	372	viking_flush_page(page);
	373	else
	374	__flush_page_to_ram(page);
	375
	376	pgdp = pgd_offset(&init_mm, addr);
	377	pmdp = pmd_offset(pgdp, addr);
	378	ptep = pte_offset_map(pmdp, addr);
	379
	380	set_pte(ptep, mk_pte(virt_to_page(page), dvma_prot));
	381	}
	382	iopte_val(*iopte++) =
	383	MKIOPTE(page_to_pfn(virt_to_page(page)), ioperm_noc);
	384	addr += PAGE_SIZE;
	385	va += PAGE_SIZE;
	386	}
	387	/* P3: why do we need this?
	388	*
	389	* DAVEM: Because there are several aspects, none of which
	390	* are handled by a single interface. Some cpus are
	391	* completely not I/O DMA coherent, and some have
	392	* virtually indexed caches. The driver DMA flushing
	393	* methods handle the former case, but here during
	394	* IOMMU page table modifications, and usage of non-cacheable
	395	* cpu mappings of pages potentially in the cpu caches, we have
	396	* to handle the latter case as well.
	397	*/
	398	flush_cache_all();
	399	iommu_flush_iotlb(first, len >> PAGE_SHIFT);
	400	flush_tlb_all();
	401	iommu_invalidate(iommu->regs);
	402
	403	*pba = iommu->start + (ioptex << PAGE_SHIFT);
	404	return 0;
	405	}
	406
4b1c5df2	407	static void iommu_unmap_dma_area(struct device *dev, unsigned long busa, int len)
1da177e4	408	{
4b1c5df2	409	struct iommu_struct *iommu = dev->archdata.iommu;
1da177e4 LT	410	iopte_t *iopte = iommu->page_table;
	411	unsigned long end;
	412	int ioptex = (busa - iommu->start) >> PAGE_SHIFT;
	413
1ae61388 ES	414	BUG_ON((busa & ~PAGE_MASK) != 0);
1ae61388 ES	415	BUG_ON((len & ~PAGE_MASK) != 0);
1da177e4 LT	416
	417	iopte += ioptex;
	418	end = busa + len;
	419	while (busa < end) {
	420	iopte_val(*iopte++) = 0;
	421	busa += PAGE_SIZE;
	422	}
	423	flush_tlb_all();
	424	iommu_invalidate(iommu->regs);
	425	bit_map_clear(&iommu->usemap, ioptex, len >> PAGE_SHIFT);
	426	}
1da177e4 LT	427	#endif
1da177e4 LT	428
d894d964 DM	429	static const struct sparc32_dma_ops iommu_dma_noflush_ops = {
	430	.get_scsi_one = iommu_get_scsi_one_noflush,
	431	.get_scsi_sgl = iommu_get_scsi_sgl_noflush,
	432	.release_scsi_one = iommu_release_scsi_one,
	433	.release_scsi_sgl = iommu_release_scsi_sgl,
	434	#ifdef CONFIG_SBUS
	435	.map_dma_area = iommu_map_dma_area,
	436	.unmap_dma_area = iommu_unmap_dma_area,
	437	#endif
	438	};
	439
	440	static const struct sparc32_dma_ops iommu_dma_gflush_ops = {
	441	.get_scsi_one = iommu_get_scsi_one_gflush,
	442	.get_scsi_sgl = iommu_get_scsi_sgl_gflush,
	443	.release_scsi_one = iommu_release_scsi_one,
	444	.release_scsi_sgl = iommu_release_scsi_sgl,
	445	#ifdef CONFIG_SBUS
	446	.map_dma_area = iommu_map_dma_area,
	447	.unmap_dma_area = iommu_unmap_dma_area,
	448	#endif
	449	};
	450
	451	static const struct sparc32_dma_ops iommu_dma_pflush_ops = {
	452	.get_scsi_one = iommu_get_scsi_one_pflush,
	453	.get_scsi_sgl = iommu_get_scsi_sgl_pflush,
	454	.release_scsi_one = iommu_release_scsi_one,
	455	.release_scsi_sgl = iommu_release_scsi_sgl,
	456	#ifdef CONFIG_SBUS
	457	.map_dma_area = iommu_map_dma_area,
	458	.unmap_dma_area = iommu_unmap_dma_area,
	459	#endif
	460	};
	461
1da177e4 LT	462	void __init ld_mmu_iommu(void)
	463	{
	464	viking_flush = (BTFIXUPVAL_CALL(flush_page_for_dma) == (unsigned long)viking_flush_page);
1da177e4 LT	465
	466	if (!BTFIXUPVAL_CALL(flush_page_for_dma)) {
	467	/* IO coherent chip */
d894d964	468	sparc32_dma_ops = &iommu_dma_noflush_ops;
1da177e4 LT	469	} else if (flush_page_for_dma_global) {
1da177e4 LT	470	/* flush_page_for_dma flushes everything, no matter of what page is it */
d894d964	471	sparc32_dma_ops = &iommu_dma_gflush_ops;
1da177e4	472	} else {
d894d964	473	sparc32_dma_ops = &iommu_dma_pflush_ops;
1da177e4	474	}
1da177e4 LT	475
	476	if (viking_mxcc_present \|\| srmmu_modtype == HyperSparc) {
	477	dvma_prot = __pgprot(SRMMU_CACHE \| SRMMU_ET_PTE \| SRMMU_PRIV);
	478	ioperm_noc = IOPTE_CACHE \| IOPTE_WRITE \| IOPTE_VALID;
	479	} else {
	480	dvma_prot = __pgprot(SRMMU_ET_PTE \| SRMMU_PRIV);
	481	ioperm_noc = IOPTE_WRITE \| IOPTE_VALID;
	482	}
	483	}