[mirror_ubuntu-zesty-kernel.git] / arch / ppc64 / kernel / pSeries_iommu.c

/*
 * arch/ppc64/kernel/pSeries_iommu.c
 *
 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
 *
 * Rewrite, cleanup: 
 *
 * Copyright (C) 2004 Olof Johansson <olof@austin.ibm.com>, IBM Corporation
 *
 * Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
 *
 * 
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
 */

#include <linux/config.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/pci.h>
#include <linux/dma-mapping.h>
#include <asm/io.h>
#include <asm/prom.h>
#include <asm/rtas.h>
#include <asm/ppcdebug.h>
#include <asm/iommu.h>
#include <asm/pci-bridge.h>
#include <asm/machdep.h>
#include <asm/abs_addr.h>
#include <asm/plpar_wrappers.h>
#include <asm/pSeries_reconfig.h>
#include <asm/systemcfg.h>
#include <asm/firmware.h>
#include "pci.h"

#define DBG(fmt...)

extern int is_python(struct device_node *);

static void tce_build_pSeries(struct iommu_table *tbl, long index, 
			      long npages, unsigned long uaddr, 
			      enum dma_data_direction direction)
{
	union tce_entry t;
	union tce_entry *tp;

	t.te_word = 0;
	t.te_rdwr = 1; // Read allowed 

	if (direction != DMA_TO_DEVICE)
		t.te_pciwr = 1;

	tp = ((union tce_entry *)tbl->it_base) + index;

	while (npages--) {
		/* can't move this out since we might cross LMB boundary */
		t.te_rpn = (virt_to_abs(uaddr)) >> PAGE_SHIFT;
	
		tp->te_word = t.te_word;

		uaddr += PAGE_SIZE;
		tp++;
	}
}


static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
{
	union tce_entry t;
	union tce_entry *tp;

	t.te_word = 0;
	tp  = ((union tce_entry *)tbl->it_base) + index;
		
	while (npages--) {
		tp->te_word = t.te_word;
		
		tp++;
	}
}


static void tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
				long npages, unsigned long uaddr,
				enum dma_data_direction direction)
{
	u64 rc;
	union tce_entry tce;

	tce.te_word = 0;
	tce.te_rpn = (virt_to_abs(uaddr)) >> PAGE_SHIFT;
	tce.te_rdwr = 1;
	if (direction != DMA_TO_DEVICE)
		tce.te_pciwr = 1;

	while (npages--) {
		rc = plpar_tce_put((u64)tbl->it_index, 
				   (u64)tcenum << 12, 
				   tce.te_word );
		
		if (rc && printk_ratelimit()) {
			printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
			printk("\tindex   = 0x%lx\n", (u64)tbl->it_index);
			printk("\ttcenum  = 0x%lx\n", (u64)tcenum);
			printk("\ttce val = 0x%lx\n", tce.te_word );
			show_stack(current, (unsigned long *)__get_SP());
		}
			
		tcenum++;
		tce.te_rpn++;
	}
}

static DEFINE_PER_CPU(void *, tce_page) = NULL;

static void tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
				     long npages, unsigned long uaddr,
				     enum dma_data_direction direction)
{
	u64 rc;
	union tce_entry tce, *tcep;
	long l, limit;

	if (npages == 1)
		return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
					   direction);

	tcep = __get_cpu_var(tce_page);

	/* This is safe to do since interrupts are off when we're called
	 * from iommu_alloc{,_sg}()
	 */
	if (!tcep) {
		tcep = (void *)__get_free_page(GFP_ATOMIC);
		/* If allocation fails, fall back to the loop implementation */
		if (!tcep)
			return tce_build_pSeriesLP(tbl, tcenum, npages,
						   uaddr, direction);
		__get_cpu_var(tce_page) = tcep;
	}

	tce.te_word = 0;
	tce.te_rpn = (virt_to_abs(uaddr)) >> PAGE_SHIFT;
	tce.te_rdwr = 1;
	if (direction != DMA_TO_DEVICE)
		tce.te_pciwr = 1;

	/* We can map max one pageful of TCEs at a time */
	do {
		/*
		 * Set up the page with TCE data, looping through and setting
		 * the values.
		 */
		limit = min_t(long, npages, PAGE_SIZE/sizeof(union tce_entry));

		for (l = 0; l < limit; l++) {
			tcep[l] = tce;
			tce.te_rpn++;
		}

		rc = plpar_tce_put_indirect((u64)tbl->it_index,
					    (u64)tcenum << 12,
					    (u64)virt_to_abs(tcep),
					    limit);

		npages -= limit;
		tcenum += limit;
	} while (npages > 0 && !rc);

	if (rc && printk_ratelimit()) {
		printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
		printk("\tindex   = 0x%lx\n", (u64)tbl->it_index);
		printk("\tnpages  = 0x%lx\n", (u64)npages);
		printk("\ttce[0] val = 0x%lx\n", tcep[0].te_word);
		show_stack(current, (unsigned long *)__get_SP());
	}
}

static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
{
	u64 rc;
	union tce_entry tce;

	tce.te_word = 0;

	while (npages--) {
		rc = plpar_tce_put((u64)tbl->it_index,
				   (u64)tcenum << 12,
				   tce.te_word);

		if (rc && printk_ratelimit()) {
			printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
			printk("\tindex   = 0x%lx\n", (u64)tbl->it_index);
			printk("\ttcenum  = 0x%lx\n", (u64)tcenum);
			printk("\ttce val = 0x%lx\n", tce.te_word );
			show_stack(current, (unsigned long *)__get_SP());
		}

		tcenum++;
	}
}


static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
{
	u64 rc;
	union tce_entry tce;

	tce.te_word = 0;

	rc = plpar_tce_stuff((u64)tbl->it_index,
			   (u64)tcenum << 12,
			   tce.te_word,
			   npages);

	if (rc && printk_ratelimit()) {
		printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
		printk("\trc      = %ld\n", rc);
		printk("\tindex   = 0x%lx\n", (u64)tbl->it_index);
		printk("\tnpages  = 0x%lx\n", (u64)npages);
		printk("\ttce val = 0x%lx\n", tce.te_word );
		show_stack(current, (unsigned long *)__get_SP());
	}
}

static void iommu_table_setparms(struct pci_controller *phb,
				 struct device_node *dn,
				 struct iommu_table *tbl) 
{
	struct device_node *node;
	unsigned long *basep;
	unsigned int *sizep;

	node = (struct device_node *)phb->arch_data;

	basep = (unsigned long *)get_property(node, "linux,tce-base", NULL);
	sizep = (unsigned int *)get_property(node, "linux,tce-size", NULL);
	if (basep == NULL || sizep == NULL) {
		printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has "
				"missing tce entries !\n", dn->full_name);
		return;
	}

	tbl->it_base = (unsigned long)__va(*basep);
	memset((void *)tbl->it_base, 0, *sizep);

	tbl->it_busno = phb->bus->number;
	
	/* Units of tce entries */
	tbl->it_offset = phb->dma_window_base_cur >> PAGE_SHIFT;
	
	/* Test if we are going over 2GB of DMA space */
	if (phb->dma_window_base_cur + phb->dma_window_size > (1L << 31))
		panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n"); 
	
	phb->dma_window_base_cur += phb->dma_window_size;

	/* Set the tce table size - measured in entries */
	tbl->it_size = phb->dma_window_size >> PAGE_SHIFT;

	tbl->it_index = 0;
	tbl->it_blocksize = 16;
	tbl->it_type = TCE_PCI;
}

/*
 * iommu_table_setparms_lpar
 *
 * Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
 *
 * ToDo: properly interpret the ibm,dma-window property.  The definition is:
 *	logical-bus-number	(1 word)
 *	phys-address		(#address-cells words)
 *	size			(#cell-size words)
 *
 * Currently we hard code these sizes (more or less).
 */
static void iommu_table_setparms_lpar(struct pci_controller *phb,
				      struct device_node *dn,
				      struct iommu_table *tbl,
				      unsigned int *dma_window)
{
	tbl->it_busno  = dn->bussubno;

	/* TODO: Parse field size properties properly. */
	tbl->it_size   = (((unsigned long)dma_window[4] << 32) |
			   (unsigned long)dma_window[5]) >> PAGE_SHIFT;
	tbl->it_offset = (((unsigned long)dma_window[2] << 32) |
			   (unsigned long)dma_window[3]) >> PAGE_SHIFT;
	tbl->it_base   = 0;
	tbl->it_index  = dma_window[0];
	tbl->it_blocksize  = 16;
	tbl->it_type = TCE_PCI;
}

static void iommu_bus_setup_pSeries(struct pci_bus *bus)
{
	struct device_node *dn, *pdn;
	struct iommu_table *tbl;

	DBG("iommu_bus_setup_pSeries, bus %p, bus->self %p\n", bus, bus->self);

	/* For each (root) bus, we carve up the available DMA space in 256MB
	 * pieces. Since each piece is used by one (sub) bus/device, that would
	 * give a maximum of 7 devices per PHB. In most cases, this is plenty.
	 *
	 * The exception is on Python PHBs (pre-POWER4). Here we don't have EADS
	 * bridges below the PHB to allocate the sectioned tables to, so instead
	 * we allocate a 1GB table at the PHB level.
	 */

	dn = pci_bus_to_OF_node(bus);

	if (!bus->self) {
		/* Root bus */
		if (is_python(dn)) {
			unsigned int *iohole;

			DBG("Python root bus %s\n", bus->name);

			iohole = (unsigned int *)get_property(dn, "io-hole", 0);

			if (iohole) {
				/* On first bus we need to leave room for the
				 * ISA address space. Just skip the first 256MB
				 * alltogether. This leaves 768MB for the window.
				 */
				DBG("PHB has io-hole, reserving 256MB\n");
				dn->phb->dma_window_size = 3 << 28;
				dn->phb->dma_window_base_cur = 1 << 28;
			} else {
				/* 1GB window by default */
				dn->phb->dma_window_size = 1 << 30;
				dn->phb->dma_window_base_cur = 0;
			}

			tbl = kmalloc(sizeof(struct iommu_table), GFP_KERNEL);

			iommu_table_setparms(dn->phb, dn, tbl);
			dn->iommu_table = iommu_init_table(tbl);
		} else {
			/* Do a 128MB table at root. This is used for the IDE
			 * controller on some SMP-mode POWER4 machines. It
			 * doesn't hurt to allocate it on other machines
			 * -- it'll just be unused since new tables are
			 * allocated on the EADS level.
			 *
			 * Allocate at offset 128MB to avoid having to deal
			 * with ISA holes; 128MB table for IDE is plenty.
			 */
			dn->phb->dma_window_size = 1 << 27;
			dn->phb->dma_window_base_cur = 1 << 27;

			tbl = kmalloc(sizeof(struct iommu_table), GFP_KERNEL);

			iommu_table_setparms(dn->phb, dn, tbl);
			dn->iommu_table = iommu_init_table(tbl);

			/* All child buses have 256MB tables */
			dn->phb->dma_window_size = 1 << 28;
		}
	} else {
		pdn = pci_bus_to_OF_node(bus->parent);

		if (!bus->parent->self && !is_python(pdn)) {
			struct iommu_table *tbl;
			/* First child and not python means this is the EADS
			 * level. Allocate new table for this slot with 256MB
			 * window.
			 */

			tbl = kmalloc(sizeof(struct iommu_table), GFP_KERNEL);

			iommu_table_setparms(dn->phb, dn, tbl);

			dn->iommu_table = iommu_init_table(tbl);
		} else {
			/* Lower than first child or under python, use parent table */
			dn->iommu_table = pdn->iommu_table;
		}
	}
}


static void iommu_bus_setup_pSeriesLP(struct pci_bus *bus)
{
	struct iommu_table *tbl;
	struct device_node *dn, *pdn;
	unsigned int *dma_window = NULL;

	DBG("iommu_bus_setup_pSeriesLP, bus %p, bus->self %p\n", bus, bus->self);

	dn = pci_bus_to_OF_node(bus);

	/* Find nearest ibm,dma-window, walking up the device tree */
	for (pdn = dn; pdn != NULL; pdn = pdn->parent) {
		dma_window = (unsigned int *)get_property(pdn, "ibm,dma-window", NULL);
		if (dma_window != NULL)
			break;
	}

	if (dma_window == NULL) {
		DBG("iommu_bus_setup_pSeriesLP: bus %s seems to have no ibm,dma-window property\n", dn->full_name);
		return;
	}

	if (!pdn->iommu_table) {
		/* Bussubno hasn't been copied yet.
		 * Do it now because iommu_table_setparms_lpar needs it.
		 */
		pdn->bussubno = bus->number;

		tbl = (struct iommu_table *)kmalloc(sizeof(struct iommu_table),
						    GFP_KERNEL);
	
		iommu_table_setparms_lpar(pdn->phb, pdn, tbl, dma_window);

		pdn->iommu_table = iommu_init_table(tbl);
	}

	if (pdn != dn)
		dn->iommu_table = pdn->iommu_table;
}


static void iommu_dev_setup_pSeries(struct pci_dev *dev)
{
	struct device_node *dn, *mydn;

	DBG("iommu_dev_setup_pSeries, dev %p (%s)\n", dev, dev->pretty_name);
	/* Now copy the iommu_table ptr from the bus device down to the
	 * pci device_node.  This means get_iommu_table() won't need to search
	 * up the device tree to find it.
	 */
	mydn = dn = pci_device_to_OF_node(dev);

	while (dn && dn->iommu_table == NULL)
		dn = dn->parent;

	if (dn) {
		mydn->iommu_table = dn->iommu_table;
	} else {
		DBG("iommu_dev_setup_pSeries, dev %p (%s) has no iommu table\n", dev, dev->pretty_name);
	}
}

static int iommu_reconfig_notifier(struct notifier_block *nb, unsigned long action, void *node)
{
	int err = NOTIFY_OK;
	struct device_node *np = node;

	switch (action) {
	case PSERIES_RECONFIG_REMOVE:
		if (np->iommu_table &&
		    get_property(np, "ibm,dma-window", NULL))
			iommu_free_table(np);
		break;
	default:
		err = NOTIFY_DONE;
		break;
	}
	return err;
}

static struct notifier_block iommu_reconfig_nb = {
	.notifier_call = iommu_reconfig_notifier,
};

static void iommu_dev_setup_pSeriesLP(struct pci_dev *dev)
{
	struct device_node *pdn, *dn;
	struct iommu_table *tbl;
	int *dma_window = NULL;

	DBG("iommu_dev_setup_pSeriesLP, dev %p (%s)\n", dev, dev->pretty_name);

	/* dev setup for LPAR is a little tricky, since the device tree might
	 * contain the dma-window properties per-device and not neccesarily
	 * for the bus. So we need to search upwards in the tree until we
	 * either hit a dma-window property, OR find a parent with a table
	 * already allocated.
	 */
	dn = pci_device_to_OF_node(dev);

	for (pdn = dn; pdn && !pdn->iommu_table; pdn = pdn->parent) {
		dma_window = (unsigned int *)get_property(pdn, "ibm,dma-window", NULL);
		if (dma_window)
			break;
	}

	/* Check for parent == NULL so we don't try to setup the empty EADS
	 * slots on POWER4 machines.
	 */
	if (dma_window == NULL || pdn->parent == NULL) {
		/* Fall back to regular (non-LPAR) dev setup */
		DBG("No dma window for device, falling back to regular setup\n");
		iommu_dev_setup_pSeries(dev);
		return;
	} else {
		DBG("Found DMA window, allocating table\n");
	}

	if (!pdn->iommu_table) {
		/* iommu_table_setparms_lpar needs bussubno. */
		pdn->bussubno = pdn->phb->bus->number;

		tbl = (struct iommu_table *)kmalloc(sizeof(struct iommu_table),
						    GFP_KERNEL);

		iommu_table_setparms_lpar(pdn->phb, pdn, tbl, dma_window);

		pdn->iommu_table = iommu_init_table(tbl);
	}

	if (pdn != dn)
		dn->iommu_table = pdn->iommu_table;
}

static void iommu_bus_setup_null(struct pci_bus *b) { }
static void iommu_dev_setup_null(struct pci_dev *d) { }

/* These are called very early. */
void iommu_init_early_pSeries(void)
{
	if (of_chosen && get_property(of_chosen, "linux,iommu-off", NULL)) {
		/* Direct I/O, IOMMU off */
		ppc_md.iommu_dev_setup = iommu_dev_setup_null;
		ppc_md.iommu_bus_setup = iommu_bus_setup_null;
		pci_direct_iommu_init();

		return;
	}

	if (systemcfg->platform & PLATFORM_LPAR) {
		if (firmware_has_feature(FW_FEATURE_MULTITCE)) {
			ppc_md.tce_build = tce_buildmulti_pSeriesLP;
			ppc_md.tce_free	 = tce_freemulti_pSeriesLP;
		} else {
			ppc_md.tce_build = tce_build_pSeriesLP;
			ppc_md.tce_free	 = tce_free_pSeriesLP;
		}
		ppc_md.iommu_bus_setup = iommu_bus_setup_pSeriesLP;
		ppc_md.iommu_dev_setup = iommu_dev_setup_pSeriesLP;
	} else {
		ppc_md.tce_build = tce_build_pSeries;
		ppc_md.tce_free  = tce_free_pSeries;
		ppc_md.iommu_bus_setup = iommu_bus_setup_pSeries;
		ppc_md.iommu_dev_setup = iommu_dev_setup_pSeries;
	}


	pSeries_reconfig_notifier_register(&iommu_reconfig_nb);

	pci_iommu_init();
}
Commit	Line	Data
1da177e4 LT	1	/*
	2	* arch/ppc64/kernel/pSeries_iommu.c
	3	*
	4	* Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
	5	*
	6	* Rewrite, cleanup:
	7	*
	8	* Copyright (C) 2004 Olof Johansson <olof@austin.ibm.com>, IBM Corporation
	9	*
	10	* Dynamic DMA mapping support, pSeries-specific parts, both SMP and LPAR.
	11	*
	12	*
	13	* This program is free software; you can redistribute it and/or modify
	14	* it under the terms of the GNU General Public License as published by
	15	* the Free Software Foundation; either version 2 of the License, or
	16	* (at your option) any later version.
	17	*
	18	* This program is distributed in the hope that it will be useful,
	19	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	20	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	21	* GNU General Public License for more details.
	22	*
	23	* You should have received a copy of the GNU General Public License
	24	* along with this program; if not, write to the Free Software
	25	* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
	26	*/
	27
	28	#include <linux/config.h>
	29	#include <linux/init.h>
	30	#include <linux/types.h>
	31	#include <linux/slab.h>
	32	#include <linux/mm.h>
	33	#include <linux/spinlock.h>
	34	#include <linux/string.h>
	35	#include <linux/pci.h>
	36	#include <linux/dma-mapping.h>
	37	#include <asm/io.h>
	38	#include <asm/prom.h>
	39	#include <asm/rtas.h>
	40	#include <asm/ppcdebug.h>
	41	#include <asm/iommu.h>
	42	#include <asm/pci-bridge.h>
	43	#include <asm/machdep.h>
	44	#include <asm/abs_addr.h>
	45	#include <asm/plpar_wrappers.h>
	46	#include <asm/pSeries_reconfig.h>
	47	#include <asm/systemcfg.h>
1ababe11	48	#include <asm/firmware.h>
1da177e4 LT	49	#include "pci.h"
	50
	51	#define DBG(fmt...)
	52
	53	extern int is_python(struct device_node *);
	54
	55	static void tce_build_pSeries(struct iommu_table *tbl, long index,
	56	long npages, unsigned long uaddr,
	57	enum dma_data_direction direction)
	58	{
	59	union tce_entry t;
	60	union tce_entry *tp;
	61
	62	t.te_word = 0;
	63	t.te_rdwr = 1; // Read allowed
	64
	65	if (direction != DMA_TO_DEVICE)
	66	t.te_pciwr = 1;
	67
	68	tp = ((union tce_entry *)tbl->it_base) + index;
	69
	70	while (npages--) {
	71	/* can't move this out since we might cross LMB boundary */
	72	t.te_rpn = (virt_to_abs(uaddr)) >> PAGE_SHIFT;
	73
	74	tp->te_word = t.te_word;
	75
	76	uaddr += PAGE_SIZE;
	77	tp++;
	78	}
	79	}
	80
	81
	82	static void tce_free_pSeries(struct iommu_table *tbl, long index, long npages)
	83	{
	84	union tce_entry t;
	85	union tce_entry *tp;
	86
	87	t.te_word = 0;
	88	tp = ((union tce_entry *)tbl->it_base) + index;
	89
	90	while (npages--) {
	91	tp->te_word = t.te_word;
	92
	93	tp++;
	94	}
	95	}
	96
	97
	98	static void tce_build_pSeriesLP(struct iommu_table *tbl, long tcenum,
	99	long npages, unsigned long uaddr,
	100	enum dma_data_direction direction)
	101	{
	102	u64 rc;
	103	union tce_entry tce;
	104
	105	tce.te_word = 0;
	106	tce.te_rpn = (virt_to_abs(uaddr)) >> PAGE_SHIFT;
	107	tce.te_rdwr = 1;
	108	if (direction != DMA_TO_DEVICE)
	109	tce.te_pciwr = 1;
	110
	111	while (npages--) {
	112	rc = plpar_tce_put((u64)tbl->it_index,
113	(u64)tcenum << 12,
114	tce.te_word );
115
116	if (rc && printk_ratelimit()) {
117	printk("tce_build_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
118	printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
119	printk("\ttcenum = 0x%lx\n", (u64)tcenum);
120	printk("\ttce val = 0x%lx\n", tce.te_word );
121	show_stack(current, (unsigned long *)__get_SP());
122	}
123
124	tcenum++;
125	tce.te_rpn++;
126	}
127	}
128
129	static DEFINE_PER_CPU(void *, tce_page) = NULL;
130
131	static void tce_buildmulti_pSeriesLP(struct iommu_table *tbl, long tcenum,
132	long npages, unsigned long uaddr,
133	enum dma_data_direction direction)
134	{
135	u64 rc;
136	union tce_entry tce, *tcep;
137	long l, limit;
138
139	if (npages == 1)
140	return tce_build_pSeriesLP(tbl, tcenum, npages, uaddr,
141	direction);
142
143	tcep = __get_cpu_var(tce_page);
144
145	/* This is safe to do since interrupts are off when we're called
146	* from iommu_alloc{,_sg}()
147	*/
148	if (!tcep) {
149	tcep = (void *)__get_free_page(GFP_ATOMIC);
150	/* If allocation fails, fall back to the loop implementation */
151	if (!tcep)
152	return tce_build_pSeriesLP(tbl, tcenum, npages,
153	uaddr, direction);
154	__get_cpu_var(tce_page) = tcep;
155	}
156
157	tce.te_word = 0;
158	tce.te_rpn = (virt_to_abs(uaddr)) >> PAGE_SHIFT;
159	tce.te_rdwr = 1;
160	if (direction != DMA_TO_DEVICE)
161	tce.te_pciwr = 1;
162
163	/* We can map max one pageful of TCEs at a time */
164	do {
165	/*
166	* Set up the page with TCE data, looping through and setting
167	* the values.
168	*/
169	limit = min_t(long, npages, PAGE_SIZE/sizeof(union tce_entry));
170
171	for (l = 0; l < limit; l++) {
172	tcep[l] = tce;
173	tce.te_rpn++;
174	}
175
176	rc = plpar_tce_put_indirect((u64)tbl->it_index,
177	(u64)tcenum << 12,
178	(u64)virt_to_abs(tcep),
179	limit);
180
181	npages -= limit;
182	tcenum += limit;
183	} while (npages > 0 && !rc);
184
185	if (rc && printk_ratelimit()) {
186	printk("tce_buildmulti_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
187	printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
188	printk("\tnpages = 0x%lx\n", (u64)npages);
189	printk("\ttce[0] val = 0x%lx\n", tcep[0].te_word);
190	show_stack(current, (unsigned long *)__get_SP());
191	}
192	}
193
194	static void tce_free_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
195	{
196	u64 rc;
197	union tce_entry tce;
198
199	tce.te_word = 0;
200
201	while (npages--) {
202	rc = plpar_tce_put((u64)tbl->it_index,
203	(u64)tcenum << 12,
204	tce.te_word);
205
206	if (rc && printk_ratelimit()) {
207	printk("tce_free_pSeriesLP: plpar_tce_put failed. rc=%ld\n", rc);
208	printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
209	printk("\ttcenum = 0x%lx\n", (u64)tcenum);
210	printk("\ttce val = 0x%lx\n", tce.te_word );
211	show_stack(current, (unsigned long *)__get_SP());
212	}
213
214	tcenum++;
215	}
216	}
217
218
219	static void tce_freemulti_pSeriesLP(struct iommu_table *tbl, long tcenum, long npages)
220	{
221	u64 rc;
222	union tce_entry tce;
223
224	tce.te_word = 0;
225
226	rc = plpar_tce_stuff((u64)tbl->it_index,
227	(u64)tcenum << 12,
228	tce.te_word,
229	npages);
230
231	if (rc && printk_ratelimit()) {
232	printk("tce_freemulti_pSeriesLP: plpar_tce_stuff failed\n");
233	printk("\trc = %ld\n", rc);
234	printk("\tindex = 0x%lx\n", (u64)tbl->it_index);
235	printk("\tnpages = 0x%lx\n", (u64)npages);
236	printk("\ttce val = 0x%lx\n", tce.te_word );
237	show_stack(current, (unsigned long *)__get_SP());
238	}
239	}
240
241	static void iommu_table_setparms(struct pci_controller *phb,
242	struct device_node *dn,
243	struct iommu_table *tbl)
244	{
245	struct device_node *node;
246	unsigned long *basep;
247	unsigned int *sizep;
248
249	node = (struct device_node *)phb->arch_data;
250
251	basep = (unsigned long *)get_property(node, "linux,tce-base", NULL);
252	sizep = (unsigned int *)get_property(node, "linux,tce-size", NULL);
253	if (basep == NULL \|\| sizep == NULL) {
254	printk(KERN_ERR "PCI_DMA: iommu_table_setparms: %s has "
255	"missing tce entries !\n", dn->full_name);
256	return;
257	}
258
259	tbl->it_base = (unsigned long)__va(*basep);
260	memset((void )tbl->it_base, 0, sizep);
261
262	tbl->it_busno = phb->bus->number;
263
264	/* Units of tce entries */
265	tbl->it_offset = phb->dma_window_base_cur >> PAGE_SHIFT;
266
267	/* Test if we are going over 2GB of DMA space */
268	if (phb->dma_window_base_cur + phb->dma_window_size > (1L << 31))
269	panic("PCI_DMA: Unexpected number of IOAs under this PHB.\n");
270
271	phb->dma_window_base_cur += phb->dma_window_size;
272
273	/* Set the tce table size - measured in entries */
274	tbl->it_size = phb->dma_window_size >> PAGE_SHIFT;
275
276	tbl->it_index = 0;
277	tbl->it_blocksize = 16;
278	tbl->it_type = TCE_PCI;
279	}
280
281	/*
282	* iommu_table_setparms_lpar
283	*
284	* Function: On pSeries LPAR systems, return TCE table info, given a pci bus.
285	*
286	* ToDo: properly interpret the ibm,dma-window property. The definition is:
287	* logical-bus-number (1 word)
288	* phys-address (#address-cells words)
289	* size (#cell-size words)
290	*
291	* Currently we hard code these sizes (more or less).
292	*/
293	static void iommu_table_setparms_lpar(struct pci_controller *phb,
294	struct device_node *dn,
295	struct iommu_table *tbl,
296	unsigned int *dma_window)
297	{
298	tbl->it_busno = dn->bussubno;
299
300	/* TODO: Parse field size properties properly. */
301	tbl->it_size = (((unsigned long)dma_window[4] << 32) \|
302	(unsigned long)dma_window[5]) >> PAGE_SHIFT;
303	tbl->it_offset = (((unsigned long)dma_window[2] << 32) \|
304	(unsigned long)dma_window[3]) >> PAGE_SHIFT;
305	tbl->it_base = 0;
306	tbl->it_index = dma_window[0];
307	tbl->it_blocksize = 16;
308	tbl->it_type = TCE_PCI;
309	}
310
311	static void iommu_bus_setup_pSeries(struct pci_bus *bus)
312	{
313	struct device_node dn, pdn;
314	struct iommu_table *tbl;
315
316	DBG("iommu_bus_setup_pSeries, bus %p, bus->self %p\n", bus, bus->self);
317
318	/* For each (root) bus, we carve up the available DMA space in 256MB
319	* pieces. Since each piece is used by one (sub) bus/device, that would
320	* give a maximum of 7 devices per PHB. In most cases, this is plenty.
321	*
322	* The exception is on Python PHBs (pre-POWER4). Here we don't have EADS
323	* bridges below the PHB to allocate the sectioned tables to, so instead
324	* we allocate a 1GB table at the PHB level.
325	*/
326
327	dn = pci_bus_to_OF_node(bus);
328
329	if (!bus->self) {
330	/* Root bus */
331	if (is_python(dn)) {
332	unsigned int *iohole;
333
334	DBG("Python root bus %s\n", bus->name);
335
336	iohole = (unsigned int *)get_property(dn, "io-hole", 0);
337
338	if (iohole) {
339	/* On first bus we need to leave room for the
340	* ISA address space. Just skip the first 256MB
341	* alltogether. This leaves 768MB for the window.
342	*/
343	DBG("PHB has io-hole, reserving 256MB\n");
344	dn->phb->dma_window_size = 3 << 28;
345	dn->phb->dma_window_base_cur = 1 << 28;
346	} else {
347	/* 1GB window by default */
348	dn->phb->dma_window_size = 1 << 30;
349	dn->phb->dma_window_base_cur = 0;
350	}
351
352	tbl = kmalloc(sizeof(struct iommu_table), GFP_KERNEL);
353
354	iommu_table_setparms(dn->phb, dn, tbl);
355	dn->iommu_table = iommu_init_table(tbl);
356	} else {
357	/* Do a 128MB table at root. This is used for the IDE
358	* controller on some SMP-mode POWER4 machines. It
359	* doesn't hurt to allocate it on other machines
360	* -- it'll just be unused since new tables are
361	* allocated on the EADS level.
362	*
363	* Allocate at offset 128MB to avoid having to deal
364	* with ISA holes; 128MB table for IDE is plenty.
365	*/
366	dn->phb->dma_window_size = 1 << 27;
367	dn->phb->dma_window_base_cur = 1 << 27;
368
369	tbl = kmalloc(sizeof(struct iommu_table), GFP_KERNEL);
370
371	iommu_table_setparms(dn->phb, dn, tbl);
372	dn->iommu_table = iommu_init_table(tbl);
373
374	/* All child buses have 256MB tables */
375	dn->phb->dma_window_size = 1 << 28;
376	}
377	} else {
378	pdn = pci_bus_to_OF_node(bus->parent);
379
380	if (!bus->parent->self && !is_python(pdn)) {
381	struct iommu_table *tbl;
382	/* First child and not python means this is the EADS
383	* level. Allocate new table for this slot with 256MB
384	* window.
385	*/
386
387	tbl = kmalloc(sizeof(struct iommu_table), GFP_KERNEL);
388
389	iommu_table_setparms(dn->phb, dn, tbl);
390
391	dn->iommu_table = iommu_init_table(tbl);
392	} else {
393	/* Lower than first child or under python, use parent table */
394	dn->iommu_table = pdn->iommu_table;
395	}
396	}
397	}
398
399
400	static void iommu_bus_setup_pSeriesLP(struct pci_bus *bus)
401	{
402	struct iommu_table *tbl;
403	struct device_node dn, pdn;
404	unsigned int *dma_window = NULL;
405
406	DBG("iommu_bus_setup_pSeriesLP, bus %p, bus->self %p\n", bus, bus->self);
407
408	dn = pci_bus_to_OF_node(bus);
409
410	/* Find nearest ibm,dma-window, walking up the device tree */
411	for (pdn = dn; pdn != NULL; pdn = pdn->parent) {
412	dma_window = (unsigned int *)get_property(pdn, "ibm,dma-window", NULL);
413	if (dma_window != NULL)
414	break;
415	}
416
417	if (dma_window == NULL) {
418	DBG("iommu_bus_setup_pSeriesLP: bus %s seems to have no ibm,dma-window property\n", dn->full_name);
419	return;
420	}
421
422	if (!pdn->iommu_table) {
423	/* Bussubno hasn't been copied yet.
424	* Do it now because iommu_table_setparms_lpar needs it.
425	*/
426	pdn->bussubno = bus->number;
427
428	tbl = (struct iommu_table *)kmalloc(sizeof(struct iommu_table),
429	GFP_KERNEL);
430
431	iommu_table_setparms_lpar(pdn->phb, pdn, tbl, dma_window);
432
433	pdn->iommu_table = iommu_init_table(tbl);
434	}
435
436	if (pdn != dn)
437	dn->iommu_table = pdn->iommu_table;
438	}
439
440
441	static void iommu_dev_setup_pSeries(struct pci_dev *dev)
442	{
443	struct device_node dn, mydn;
444
445	DBG("iommu_dev_setup_pSeries, dev %p (%s)\n", dev, dev->pretty_name);
446	/* Now copy the iommu_table ptr from the bus device down to the
447	* pci device_node. This means get_iommu_table() won't need to search
448	* up the device tree to find it.
449	*/
450	mydn = dn = pci_device_to_OF_node(dev);
451
452	while (dn && dn->iommu_table == NULL)
453	dn = dn->parent;
454
455	if (dn) {
456	mydn->iommu_table = dn->iommu_table;
457	} else {
458	DBG("iommu_dev_setup_pSeries, dev %p (%s) has no iommu table\n", dev, dev->pretty_name);
459	}
460	}
461
462	static int iommu_reconfig_notifier(struct notifier_block nb, unsigned long action, void node)
463	{
464	int err = NOTIFY_OK;
465	struct device_node *np = node;
466
467	switch (action) {
468	case PSERIES_RECONFIG_REMOVE:
469	if (np->iommu_table &&
470	get_property(np, "ibm,dma-window", NULL))
471	iommu_free_table(np);
472	break;
473	default:
474	err = NOTIFY_DONE;
475	break;
476	}
477	return err;
478	}
479
480	static struct notifier_block iommu_reconfig_nb = {
481	.notifier_call = iommu_reconfig_notifier,
482	};
483
484	static void iommu_dev_setup_pSeriesLP(struct pci_dev *dev)
485	{
486	struct device_node pdn, dn;
487	struct iommu_table *tbl;
488	int *dma_window = NULL;
489
490	DBG("iommu_dev_setup_pSeriesLP, dev %p (%s)\n", dev, dev->pretty_name);
491
492	/* dev setup for LPAR is a little tricky, since the device tree might
493	* contain the dma-window properties per-device and not neccesarily
494	* for the bus. So we need to search upwards in the tree until we
495	* either hit a dma-window property, OR find a parent with a table
496	* already allocated.
497	*/
498	dn = pci_device_to_OF_node(dev);
499
500	for (pdn = dn; pdn && !pdn->iommu_table; pdn = pdn->parent) {
501	dma_window = (unsigned int *)get_property(pdn, "ibm,dma-window", NULL);
502	if (dma_window)
503	break;
504	}
505
506	/* Check for parent == NULL so we don't try to setup the empty EADS
507	* slots on POWER4 machines.
508	*/
509	if (dma_window == NULL \|\| pdn->parent == NULL) {
510	/* Fall back to regular (non-LPAR) dev setup */
511	DBG("No dma window for device, falling back to regular setup\n");
512	iommu_dev_setup_pSeries(dev);
513	return;
514	} else {
515	DBG("Found DMA window, allocating table\n");
516	}
517
518	if (!pdn->iommu_table) {
519	/* iommu_table_setparms_lpar needs bussubno. */
520	pdn->bussubno = pdn->phb->bus->number;
521
522	tbl = (struct iommu_table *)kmalloc(sizeof(struct iommu_table),
523	GFP_KERNEL);
524
525	iommu_table_setparms_lpar(pdn->phb, pdn, tbl, dma_window);
526
527	pdn->iommu_table = iommu_init_table(tbl);
528	}
529
530	if (pdn != dn)
531	dn->iommu_table = pdn->iommu_table;
532	}
533
534	static void iommu_bus_setup_null(struct pci_bus *b) { }
535	static void iommu_dev_setup_null(struct pci_dev *d) { }
536
537	/* These are called very early. */
538	void iommu_init_early_pSeries(void)
539	{
540	if (of_chosen && get_property(of_chosen, "linux,iommu-off", NULL)) {
541	/* Direct I/O, IOMMU off */
542	ppc_md.iommu_dev_setup = iommu_dev_setup_null;
543	ppc_md.iommu_bus_setup = iommu_bus_setup_null;
544	pci_direct_iommu_init();
545
546	return;
547	}
548
549	if (systemcfg->platform & PLATFORM_LPAR) {
1ababe11	550	if (firmware_has_feature(FW_FEATURE_MULTITCE)) {
1da177e4 LT	551	ppc_md.tce_build = tce_buildmulti_pSeriesLP;
	552	ppc_md.tce_free = tce_freemulti_pSeriesLP;
	553	} else {
	554	ppc_md.tce_build = tce_build_pSeriesLP;
	555	ppc_md.tce_free = tce_free_pSeriesLP;
	556	}
	557	ppc_md.iommu_bus_setup = iommu_bus_setup_pSeriesLP;
	558	ppc_md.iommu_dev_setup = iommu_dev_setup_pSeriesLP;
	559	} else {
	560	ppc_md.tce_build = tce_build_pSeries;
	561	ppc_md.tce_free = tce_free_pSeries;
	562	ppc_md.iommu_bus_setup = iommu_bus_setup_pSeries;
	563	ppc_md.iommu_dev_setup = iommu_dev_setup_pSeries;
	564	}
	565
	566
	567	pSeries_reconfig_notifier_register(&iommu_reconfig_nb);
	568
	569	pci_iommu_init();
	570	}
	571