[mirror_ubuntu-hirsute-kernel.git] / arch / x86 / kernel / pci-dma.c

#include <linux/dma-mapping.h>
#include <linux/dmar.h>
#include <linux/bootmem.h>
#include <linux/pci.h>

#include <asm/proto.h>
#include <asm/dma.h>
#include <asm/iommu.h>
#include <asm/gart.h>
#include <asm/calgary.h>
#include <asm/amd_iommu.h>

static int forbid_dac __read_mostly;

struct dma_mapping_ops *dma_ops;
EXPORT_SYMBOL(dma_ops);

static int iommu_sac_force __read_mostly;

#ifdef CONFIG_IOMMU_DEBUG
int panic_on_overflow __read_mostly = 1;
int force_iommu __read_mostly = 1;
#else
int panic_on_overflow __read_mostly = 0;
int force_iommu __read_mostly = 0;
#endif

int iommu_merge __read_mostly = 0;

int no_iommu __read_mostly;
/* Set this to 1 if there is a HW IOMMU in the system */
int iommu_detected __read_mostly = 0;

/* This tells the BIO block layer to assume merging. Default to off
   because we cannot guarantee merging later. */
int iommu_bio_merge __read_mostly = 0;
EXPORT_SYMBOL(iommu_bio_merge);

dma_addr_t bad_dma_address __read_mostly = 0;
EXPORT_SYMBOL(bad_dma_address);

/* Dummy device used for NULL arguments (normally ISA). Better would
   be probably a smaller DMA mask, but this is bug-to-bug compatible
   to older i386. */
struct device x86_dma_fallback_dev = {
	.bus_id = "fallback device",
	.coherent_dma_mask = DMA_32BIT_MASK,
	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
};
EXPORT_SYMBOL(x86_dma_fallback_dev);

int dma_set_mask(struct device *dev, u64 mask)
{
	if (!dev->dma_mask || !dma_supported(dev, mask))
		return -EIO;

	*dev->dma_mask = mask;

	return 0;
}
EXPORT_SYMBOL(dma_set_mask);

#ifdef CONFIG_X86_64
static __initdata void *dma32_bootmem_ptr;
static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);

static int __init parse_dma32_size_opt(char *p)
{
	if (!p)
		return -EINVAL;
	dma32_bootmem_size = memparse(p, &p);
	return 0;
}
early_param("dma32_size", parse_dma32_size_opt);

void __init dma32_reserve_bootmem(void)
{
	unsigned long size, align;
	if (max_pfn <= MAX_DMA32_PFN)
		return;

	/*
	 * check aperture_64.c allocate_aperture() for reason about
	 * using 512M as goal
	 */
	align = 64ULL<<20;
	size = roundup(dma32_bootmem_size, align);
	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
				 512ULL<<20);
	if (dma32_bootmem_ptr)
		dma32_bootmem_size = size;
	else
		dma32_bootmem_size = 0;
}
static void __init dma32_free_bootmem(void)
{

	if (max_pfn <= MAX_DMA32_PFN)
		return;

	if (!dma32_bootmem_ptr)
		return;

	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);

	dma32_bootmem_ptr = NULL;
	dma32_bootmem_size = 0;
}

void __init pci_iommu_alloc(void)
{
	/* free the range so iommu could get some range less than 4G */
	dma32_free_bootmem();
	/*
	 * The order of these functions is important for
	 * fall-back/fail-over reasons
	 */
	gart_iommu_hole_init();

	detect_calgary();

	detect_intel_iommu();

	amd_iommu_detect();

	pci_swiotlb_init();
}

unsigned long iommu_nr_pages(unsigned long addr, unsigned long len)
{
	unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);

	return size >> PAGE_SHIFT;
}
EXPORT_SYMBOL(iommu_nr_pages);
#endif

void *dma_generic_alloc_coherent(struct device *dev, size_t size,
				 dma_addr_t *dma_addr, gfp_t flag)
{
	unsigned long dma_mask;
	struct page *page;
	dma_addr_t addr;

	dma_mask = dma_alloc_coherent_mask(dev, flag);

	flag |= __GFP_ZERO;
again:
	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
	if (!page)
		return NULL;

	addr = page_to_phys(page);
	if (!is_buffer_dma_capable(dma_mask, addr, size)) {
		__free_pages(page, get_order(size));

		if (dma_mask < DMA_32BIT_MASK && !(flag & GFP_DMA)) {
			flag = (flag & ~GFP_DMA32) | GFP_DMA;
			goto again;
		}

		return NULL;
	}

	*dma_addr = addr;
	return page_address(page);
}

/*
 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
 * documentation.
 */
static __init int iommu_setup(char *p)
{
	iommu_merge = 1;

	if (!p)
		return -EINVAL;

	while (*p) {
		if (!strncmp(p, "off", 3))
			no_iommu = 1;
		/* gart_parse_options has more force support */
		if (!strncmp(p, "force", 5))
			force_iommu = 1;
		if (!strncmp(p, "noforce", 7)) {
			iommu_merge = 0;
			force_iommu = 0;
		}

		if (!strncmp(p, "biomerge", 8)) {
			iommu_bio_merge = 4096;
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "panic", 5))
			panic_on_overflow = 1;
		if (!strncmp(p, "nopanic", 7))
			panic_on_overflow = 0;
		if (!strncmp(p, "merge", 5)) {
			iommu_merge = 1;
			force_iommu = 1;
		}
		if (!strncmp(p, "nomerge", 7))
			iommu_merge = 0;
		if (!strncmp(p, "forcesac", 8))
			iommu_sac_force = 1;
		if (!strncmp(p, "allowdac", 8))
			forbid_dac = 0;
		if (!strncmp(p, "nodac", 5))
			forbid_dac = -1;
		if (!strncmp(p, "usedac", 6)) {
			forbid_dac = -1;
			return 1;
		}
#ifdef CONFIG_SWIOTLB
		if (!strncmp(p, "soft", 4))
			swiotlb = 1;
#endif

		gart_parse_options(p);

#ifdef CONFIG_CALGARY_IOMMU
		if (!strncmp(p, "calgary", 7))
			use_calgary = 1;
#endif /* CONFIG_CALGARY_IOMMU */

		p += strcspn(p, ",");
		if (*p == ',')
			++p;
	}
	return 0;
}
early_param("iommu", iommu_setup);

int dma_supported(struct device *dev, u64 mask)
{
	struct dma_mapping_ops *ops = get_dma_ops(dev);

#ifdef CONFIG_PCI
	if (mask > 0xffffffff && forbid_dac > 0) {
		dev_info(dev, "PCI: Disallowing DAC for device\n");
		return 0;
	}
#endif

	if (ops->dma_supported)
		return ops->dma_supported(dev, mask);

	/* Copied from i386. Doesn't make much sense, because it will
	   only work for pci_alloc_coherent.
	   The caller just has to use GFP_DMA in this case. */
	if (mask < DMA_24BIT_MASK)
		return 0;

	/* Tell the device to use SAC when IOMMU force is on.  This
	   allows the driver to use cheaper accesses in some cases.

	   Problem with this is that if we overflow the IOMMU area and
	   return DAC as fallback address the device may not handle it
	   correctly.

	   As a special case some controllers have a 39bit address
	   mode that is as efficient as 32bit (aic79xx). Don't force
	   SAC for these.  Assume all masks <= 40 bits are of this
	   type. Normally this doesn't make any difference, but gives
	   more gentle handling of IOMMU overflow. */
	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
		dev_info(dev, "Force SAC with mask %Lx\n", mask);
		return 0;
	}

	return 1;
}
EXPORT_SYMBOL(dma_supported);

static int __init pci_iommu_init(void)
{
	calgary_iommu_init();

	intel_iommu_init();

	amd_iommu_init();

	gart_iommu_init();

	no_iommu_init();
	return 0;
}

void pci_iommu_shutdown(void)
{
	gart_iommu_shutdown();
}
/* Must execute after PCI subsystem */
fs_initcall(pci_iommu_init);

#ifdef CONFIG_PCI
/* Many VIA bridges seem to corrupt data for DAC. Disable it here */

static __devinit void via_no_dac(struct pci_dev *dev)
{
	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
		printk(KERN_INFO "PCI: VIA PCI bridge detected."
				 "Disabling DAC.\n");
		forbid_dac = 1;
	}
}
DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
#endif
Commit	Line	Data
459121c9	1	#include <linux/dma-mapping.h>
cb5867a5	2	#include <linux/dmar.h>
116890d5	3	#include <linux/bootmem.h>
bca5c096	4	#include <linux/pci.h>
cb5867a5	5
116890d5 GC	6	#include <asm/proto.h>
116890d5 GC	7	#include <asm/dma.h>
46a7fa27	8	#include <asm/iommu.h>
1d9b16d1	9	#include <asm/gart.h>
cb5867a5	10	#include <asm/calgary.h>
a69ca340	11	#include <asm/amd_iommu.h>
459121c9	12
3b15e581 FY	13	static int forbid_dac __read_mostly;
3b15e581 FY	14
8d8bb39b	15	struct dma_mapping_ops *dma_ops;
85c246ee GC	16	EXPORT_SYMBOL(dma_ops);
85c246ee GC	17
b4cdc430	18	static int iommu_sac_force __read_mostly;
8e0c3797	19
f9c258de GC	20	#ifdef CONFIG_IOMMU_DEBUG
	21	int panic_on_overflow __read_mostly = 1;
	22	int force_iommu __read_mostly = 1;
	23	#else
	24	int panic_on_overflow __read_mostly = 0;
	25	int force_iommu __read_mostly = 0;
	26	#endif
	27
fae9a0d8 GC	28	int iommu_merge __read_mostly = 0;
	29
	30	int no_iommu __read_mostly;
	31	/* Set this to 1 if there is a HW IOMMU in the system */
	32	int iommu_detected __read_mostly = 0;
	33
	34	/* This tells the BIO block layer to assume merging. Default to off
	35	because we cannot guarantee merging later. */
	36	int iommu_bio_merge __read_mostly = 0;
	37	EXPORT_SYMBOL(iommu_bio_merge);
	38
cac67877 GC	39	dma_addr_t bad_dma_address __read_mostly = 0;
cac67877 GC	40	EXPORT_SYMBOL(bad_dma_address);
fae9a0d8	41
098cb7f2 GC	42	/* Dummy device used for NULL arguments (normally ISA). Better would
	43	be probably a smaller DMA mask, but this is bug-to-bug compatible
	44	to older i386. */
6c505ce3	45	struct device x86_dma_fallback_dev = {
098cb7f2 GC	46	.bus_id = "fallback device",
098cb7f2 GC	47	.coherent_dma_mask = DMA_32BIT_MASK,
6c505ce3	48	.dma_mask = &x86_dma_fallback_dev.coherent_dma_mask,
098cb7f2	49	};
6c505ce3	50	EXPORT_SYMBOL(x86_dma_fallback_dev);
098cb7f2	51
459121c9 GC	52	int dma_set_mask(struct device *dev, u64 mask)
	53	{
	54	if (!dev->dma_mask \|\| !dma_supported(dev, mask))
	55	return -EIO;
	56
	57	*dev->dma_mask = mask;
	58
	59	return 0;
	60	}
	61	EXPORT_SYMBOL(dma_set_mask);
	62
116890d5 GC	63	#ifdef CONFIG_X86_64
	64	static __initdata void *dma32_bootmem_ptr;
	65	static unsigned long dma32_bootmem_size __initdata = (128ULL<<20);
	66
	67	static int __init parse_dma32_size_opt(char *p)
	68	{
	69	if (!p)
	70	return -EINVAL;
	71	dma32_bootmem_size = memparse(p, &p);
	72	return 0;
	73	}
	74	early_param("dma32_size", parse_dma32_size_opt);
	75
	76	void __init dma32_reserve_bootmem(void)
	77	{
	78	unsigned long size, align;
c987d12f	79	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	80	return;
116890d5 GC	81
7677b2ef YL	82	/*
	83	* check aperture_64.c allocate_aperture() for reason about
	84	* using 512M as goal
	85	*/
116890d5	86	align = 64ULL<<20;
1ddb5518	87	size = roundup(dma32_bootmem_size, align);
116890d5	88	dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align,
7677b2ef	89	512ULL<<20);
116890d5 GC	90	if (dma32_bootmem_ptr)
	91	dma32_bootmem_size = size;
	92	else
	93	dma32_bootmem_size = 0;
	94	}
	95	static void __init dma32_free_bootmem(void)
	96	{
116890d5	97
c987d12f	98	if (max_pfn <= MAX_DMA32_PFN)
116890d5 GC	99	return;
	100
	101	if (!dma32_bootmem_ptr)
	102	return;
	103
330fce23	104	free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size);
116890d5 GC	105
	106	dma32_bootmem_ptr = NULL;
	107	dma32_bootmem_size = 0;
	108	}
	109
	110	void __init pci_iommu_alloc(void)
	111	{
	112	/* free the range so iommu could get some range less than 4G */
	113	dma32_free_bootmem();
	114	/*
	115	* The order of these functions is important for
	116	* fall-back/fail-over reasons
	117	*/
116890d5	118	gart_iommu_hole_init();
116890d5	119
116890d5	120	detect_calgary();
116890d5 GC	121
	122	detect_intel_iommu();
	123
a69ca340 JR	124	amd_iommu_detect();
a69ca340 JR	125
116890d5	126	pci_swiotlb_init();
116890d5	127	}
8978b742	128
bdab0ba3	129	unsigned long iommu_nr_pages(unsigned long addr, unsigned long len)
8978b742 FT	130	{
	131	unsigned long size = roundup((addr & ~PAGE_MASK) + len, PAGE_SIZE);
	132
	133	return size >> PAGE_SHIFT;
	134	}
bdab0ba3	135	EXPORT_SYMBOL(iommu_nr_pages);
116890d5 GC	136	#endif
116890d5 GC	137
9f6ac577 FT	138	void dma_generic_alloc_coherent(struct device dev, size_t size,
	139	dma_addr_t *dma_addr, gfp_t flag)
	140	{
	141	unsigned long dma_mask;
	142	struct page *page;
	143	dma_addr_t addr;
	144
	145	dma_mask = dma_alloc_coherent_mask(dev, flag);
	146
	147	flag \|= __GFP_ZERO;
	148	again:
	149	page = alloc_pages_node(dev_to_node(dev), flag, get_order(size));
	150	if (!page)
	151	return NULL;
	152
	153	addr = page_to_phys(page);
	154	if (!is_buffer_dma_capable(dma_mask, addr, size)) {
	155	__free_pages(page, get_order(size));
	156
	157	if (dma_mask < DMA_32BIT_MASK && !(flag & GFP_DMA)) {
	158	flag = (flag & ~GFP_DMA32) \| GFP_DMA;
	159	goto again;
	160	}
	161
	162	return NULL;
	163	}
	164
	165	*dma_addr = addr;
	166	return page_address(page);
	167	}
	168
fae9a0d8 GC	169	/*
	170	* See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter
	171	* documentation.
	172	*/
	173	static __init int iommu_setup(char *p)
	174	{
	175	iommu_merge = 1;
	176
	177	if (!p)
	178	return -EINVAL;
	179
	180	while (*p) {
	181	if (!strncmp(p, "off", 3))
	182	no_iommu = 1;
	183	/* gart_parse_options has more force support */
	184	if (!strncmp(p, "force", 5))
	185	force_iommu = 1;
	186	if (!strncmp(p, "noforce", 7)) {
	187	iommu_merge = 0;
	188	force_iommu = 0;
	189	}
	190
	191	if (!strncmp(p, "biomerge", 8)) {
	192	iommu_bio_merge = 4096;
	193	iommu_merge = 1;
	194	force_iommu = 1;
	195	}
	196	if (!strncmp(p, "panic", 5))
	197	panic_on_overflow = 1;
	198	if (!strncmp(p, "nopanic", 7))
	199	panic_on_overflow = 0;
	200	if (!strncmp(p, "merge", 5)) {
	201	iommu_merge = 1;
	202	force_iommu = 1;
	203	}
	204	if (!strncmp(p, "nomerge", 7))
	205	iommu_merge = 0;
	206	if (!strncmp(p, "forcesac", 8))
	207	iommu_sac_force = 1;
	208	if (!strncmp(p, "allowdac", 8))
	209	forbid_dac = 0;
	210	if (!strncmp(p, "nodac", 5))
	211	forbid_dac = -1;
	212	if (!strncmp(p, "usedac", 6)) {
	213	forbid_dac = -1;
	214	return 1;
	215	}
	216	#ifdef CONFIG_SWIOTLB
	217	if (!strncmp(p, "soft", 4))
	218	swiotlb = 1;
	219	#endif
	220
fae9a0d8	221	gart_parse_options(p);
fae9a0d8 GC	222
	223	#ifdef CONFIG_CALGARY_IOMMU
	224	if (!strncmp(p, "calgary", 7))
	225	use_calgary = 1;
	226	#endif /* CONFIG_CALGARY_IOMMU */
	227
	228	p += strcspn(p, ",");
	229	if (*p == ',')
	230	++p;
	231	}
	232	return 0;
	233	}
	234	early_param("iommu", iommu_setup);
	235
8e0c3797 GC	236	int dma_supported(struct device *dev, u64 mask)
8e0c3797 GC	237	{
8d8bb39b FT	238	struct dma_mapping_ops *ops = get_dma_ops(dev);
8d8bb39b FT	239
8e0c3797 GC	240	#ifdef CONFIG_PCI
8e0c3797 GC	241	if (mask > 0xffffffff && forbid_dac > 0) {
fc3a8828	242	dev_info(dev, "PCI: Disallowing DAC for device\n");
8e0c3797 GC	243	return 0;
	244	}
	245	#endif
	246
8d8bb39b FT	247	if (ops->dma_supported)
8d8bb39b FT	248	return ops->dma_supported(dev, mask);
8e0c3797 GC	249
	250	/* Copied from i386. Doesn't make much sense, because it will
	251	only work for pci_alloc_coherent.
	252	The caller just has to use GFP_DMA in this case. */
	253	if (mask < DMA_24BIT_MASK)
	254	return 0;
	255
	256	/* Tell the device to use SAC when IOMMU force is on. This
	257	allows the driver to use cheaper accesses in some cases.
	258
	259	Problem with this is that if we overflow the IOMMU area and
	260	return DAC as fallback address the device may not handle it
	261	correctly.
	262
	263	As a special case some controllers have a 39bit address
	264	mode that is as efficient as 32bit (aic79xx). Don't force
	265	SAC for these. Assume all masks <= 40 bits are of this
	266	type. Normally this doesn't make any difference, but gives
	267	more gentle handling of IOMMU overflow. */
	268	if (iommu_sac_force && (mask >= DMA_40BIT_MASK)) {
fc3a8828	269	dev_info(dev, "Force SAC with mask %Lx\n", mask);
8e0c3797 GC	270	return 0;
	271	}
	272
	273	return 1;
	274	}
	275	EXPORT_SYMBOL(dma_supported);
	276
cb5867a5 GC	277	static int __init pci_iommu_init(void)
cb5867a5 GC	278	{
cb5867a5	279	calgary_iommu_init();
cb5867a5 GC	280
	281	intel_iommu_init();
	282
a69ca340 JR	283	amd_iommu_init();
a69ca340 JR	284
cb5867a5	285	gart_iommu_init();
459121c9	286
cb5867a5 GC	287	no_iommu_init();
	288	return 0;
	289	}
	290
	291	void pci_iommu_shutdown(void)
	292	{
	293	gart_iommu_shutdown();
	294	}
	295	/* Must execute after PCI subsystem */
	296	fs_initcall(pci_iommu_init);
3b15e581 FY	297
	298	#ifdef CONFIG_PCI
	299	/* Many VIA bridges seem to corrupt data for DAC. Disable it here */
	300
	301	static __devinit void via_no_dac(struct pci_dev *dev)
	302	{
	303	if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) {
	304	printk(KERN_INFO "PCI: VIA PCI bridge detected."
	305	"Disabling DAC.\n");
	306	forbid_dac = 1;
	307	}
	308	}
	309	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac);
	310	#endif