2 * arch/powerpc/sysdev/dart_iommu.c
4 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
5 * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>,
8 * Based on pSeries_iommu.c:
9 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
10 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
12 * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu.
15 * This program is free software; you can redistribute it and/or modify
16 * it under the terms of the GNU General Public License as published by
17 * the Free Software Foundation; either version 2 of the License, or
18 * (at your option) any later version.
20 * This program is distributed in the hope that it will be useful,
21 * but WITHOUT ANY WARRANTY; without even the implied warranty of
22 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23 * GNU General Public License for more details.
25 * You should have received a copy of the GNU General Public License
26 * along with this program; if not, write to the Free Software
27 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
30 #include <linux/init.h>
31 #include <linux/types.h>
33 #include <linux/spinlock.h>
34 #include <linux/string.h>
35 #include <linux/pci.h>
36 #include <linux/dma-mapping.h>
37 #include <linux/vmalloc.h>
38 #include <linux/suspend.h>
39 #include <linux/memblock.h>
40 #include <linux/gfp.h>
41 #include <linux/kmemleak.h>
44 #include <asm/iommu.h>
45 #include <asm/pci-bridge.h>
46 #include <asm/machdep.h>
47 #include <asm/cacheflush.h>
48 #include <asm/ppc-pci.h>
52 /* DART table address and size */
53 static u32
*dart_tablebase
;
54 static unsigned long dart_tablesize
;
56 /* Mapped base address for the dart */
57 static unsigned int __iomem
*dart
;
59 /* Dummy val that entries are set to when unused */
60 static unsigned int dart_emptyval
;
62 static struct iommu_table iommu_table_dart
;
63 static int iommu_table_dart_inited
;
64 static int dart_dirty
;
65 static int dart_is_u4
;
67 #define DART_U4_BYPASS_BASE 0x8000000000ull
71 static DEFINE_SPINLOCK(invalidate_lock
);
73 static inline void dart_tlb_invalidate_all(void)
76 unsigned int reg
, inv_bit
;
80 spin_lock_irqsave(&invalidate_lock
, flags
);
84 /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the
85 * control register and wait for it to clear.
87 * Gotcha: Sometimes, the DART won't detect that the bit gets
88 * set. If so, clear it and set it again.
93 inv_bit
= dart_is_u4
? DART_CNTL_U4_FLUSHTLB
: DART_CNTL_U3_FLUSHTLB
;
96 reg
= DART_IN(DART_CNTL
);
98 DART_OUT(DART_CNTL
, reg
);
100 while ((DART_IN(DART_CNTL
) & inv_bit
) && l
< (1L << limit
))
102 if (l
== (1L << limit
)) {
105 reg
= DART_IN(DART_CNTL
);
107 DART_OUT(DART_CNTL
, reg
);
110 panic("DART: TLB did not flush after waiting a long "
114 spin_unlock_irqrestore(&invalidate_lock
, flags
);
117 static inline void dart_tlb_invalidate_one(unsigned long bus_rpn
)
120 unsigned int l
, limit
;
123 spin_lock_irqsave(&invalidate_lock
, flags
);
125 reg
= DART_CNTL_U4_ENABLE
| DART_CNTL_U4_IONE
|
126 (bus_rpn
& DART_CNTL_U4_IONE_MASK
);
127 DART_OUT(DART_CNTL
, reg
);
132 while ((DART_IN(DART_CNTL
) & DART_CNTL_U4_IONE
) && l
< (1L << limit
)) {
137 if (l
== (1L << limit
)) {
142 panic("DART: TLB did not flush after waiting a long "
146 spin_unlock_irqrestore(&invalidate_lock
, flags
);
149 static void dart_cache_sync(unsigned int *base
, unsigned int count
)
152 * We add 1 to the number of entries to flush, following a
153 * comment in Darwin indicating that the memory controller
154 * can prefetch unmapped memory under some circumstances.
156 unsigned long start
= (unsigned long)base
;
157 unsigned long end
= start
+ (count
+ 1) * sizeof(unsigned int);
160 /* Perform a standard cache flush */
161 flush_inval_dcache_range(start
, end
);
164 * Perform the sequence described in the CPC925 manual to
165 * ensure all the data gets to a point the cache incoherent
166 * DART hardware will see.
168 asm volatile(" sync;"
174 " isync" : "=r" (tmp
) : "r" (end
) : "memory");
177 static void dart_flush(struct iommu_table
*tbl
)
181 dart_tlb_invalidate_all();
186 static int dart_build(struct iommu_table
*tbl
, long index
,
187 long npages
, unsigned long uaddr
,
188 enum dma_data_direction direction
,
191 unsigned int *dp
, *orig_dp
;
195 DBG("dart: build at: %lx, %lx, addr: %x\n", index
, npages
, uaddr
);
197 orig_dp
= dp
= ((unsigned int*)tbl
->it_base
) + index
;
199 /* On U3, all memory is contiguous, so we can move this
204 rpn
= __pa(uaddr
) >> DART_PAGE_SHIFT
;
206 *(dp
++) = DARTMAP_VALID
| (rpn
& DARTMAP_RPNMASK
);
208 uaddr
+= DART_PAGE_SIZE
;
210 dart_cache_sync(orig_dp
, npages
);
215 dart_tlb_invalidate_one(rpn
++);
223 static void dart_free(struct iommu_table
*tbl
, long index
, long npages
)
225 unsigned int *dp
, *orig_dp
;
226 long orig_npages
= npages
;
228 /* We don't worry about flushing the TLB cache. The only drawback of
229 * not doing it is that we won't catch buggy device drivers doing
230 * bad DMAs, but then no 32-bit architecture ever does either.
233 DBG("dart: free at: %lx, %lx\n", index
, npages
);
235 orig_dp
= dp
= ((unsigned int *)tbl
->it_base
) + index
;
238 *(dp
++) = dart_emptyval
;
240 dart_cache_sync(orig_dp
, orig_npages
);
243 static void allocate_dart(void)
247 /* 512 pages (2MB) is max DART tablesize. */
248 dart_tablesize
= 1UL << 21;
251 * 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we
252 * will blow up an entire large page anyway in the kernel mapping.
254 dart_tablebase
= memblock_alloc_try_nid_raw(SZ_16M
, SZ_16M
,
255 MEMBLOCK_LOW_LIMIT
, SZ_2G
,
258 panic("Failed to allocate 16MB below 2GB for DART table\n");
260 /* There is no point scanning the DART space for leaks*/
261 kmemleak_no_scan((void *)dart_tablebase
);
263 /* Allocate a spare page to map all invalid DART pages. We need to do
264 * that to work around what looks like a problem with the HT bridge
265 * prefetching into invalid pages and corrupting data
267 tmp
= memblock_phys_alloc(DART_PAGE_SIZE
, DART_PAGE_SIZE
);
269 panic("DART: table allocation failed\n");
271 dart_emptyval
= DARTMAP_VALID
| ((tmp
>> DART_PAGE_SHIFT
) &
274 printk(KERN_INFO
"DART table allocated at: %p\n", dart_tablebase
);
277 static int __init
dart_init(struct device_node
*dart_node
)
280 unsigned long base
, size
;
283 /* IOMMU disabled by the user ? bail out */
288 * Only use the DART if the machine has more than 1GB of RAM
289 * or if requested with iommu=on on cmdline.
291 * 1GB of RAM is picked as limit because some default devices
292 * (i.e. Airport Extreme) have 30 bit address range limits.
295 if (!iommu_force_on
&& memblock_end_of_DRAM() <= 0x40000000ull
)
298 /* Get DART registers */
299 if (of_address_to_resource(dart_node
, 0, &r
))
300 panic("DART: can't get register base ! ");
302 /* Map in DART registers */
303 dart
= ioremap(r
.start
, resource_size(&r
));
305 panic("DART: Cannot map registers!");
307 /* Allocate the DART and dummy page */
310 /* Fill initial table */
311 for (i
= 0; i
< dart_tablesize
/4; i
++)
312 dart_tablebase
[i
] = dart_emptyval
;
315 dart_cache_sync(dart_tablebase
, dart_tablesize
/ sizeof(u32
));
317 /* Initialize DART with table base and enable it. */
318 base
= ((unsigned long)dart_tablebase
) >> DART_PAGE_SHIFT
;
319 size
= dart_tablesize
>> DART_PAGE_SHIFT
;
321 size
&= DART_SIZE_U4_SIZE_MASK
;
322 DART_OUT(DART_BASE_U4
, base
);
323 DART_OUT(DART_SIZE_U4
, size
);
324 DART_OUT(DART_CNTL
, DART_CNTL_U4_ENABLE
);
326 size
&= DART_CNTL_U3_SIZE_MASK
;
328 DART_CNTL_U3_ENABLE
|
329 (base
<< DART_CNTL_U3_BASE_SHIFT
) |
330 (size
<< DART_CNTL_U3_SIZE_SHIFT
));
333 /* Invalidate DART to get rid of possible stale TLBs */
334 dart_tlb_invalidate_all();
336 printk(KERN_INFO
"DART IOMMU initialized for %s type chipset\n",
337 dart_is_u4
? "U4" : "U3");
342 static struct iommu_table_ops iommu_dart_ops
= {
348 static void iommu_table_dart_setup(void)
350 iommu_table_dart
.it_busno
= 0;
351 iommu_table_dart
.it_offset
= 0;
352 /* it_size is in number of entries */
353 iommu_table_dart
.it_size
= dart_tablesize
/ sizeof(u32
);
354 iommu_table_dart
.it_page_shift
= IOMMU_PAGE_SHIFT_4K
;
356 /* Initialize the common IOMMU code */
357 iommu_table_dart
.it_base
= (unsigned long)dart_tablebase
;
358 iommu_table_dart
.it_index
= 0;
359 iommu_table_dart
.it_blocksize
= 1;
360 iommu_table_dart
.it_ops
= &iommu_dart_ops
;
361 iommu_init_table(&iommu_table_dart
, -1);
363 /* Reserve the last page of the DART to avoid possible prefetch
364 * past the DART mapped area
366 set_bit(iommu_table_dart
.it_size
- 1, iommu_table_dart
.it_map
);
369 static void pci_dma_bus_setup_dart(struct pci_bus
*bus
)
371 if (!iommu_table_dart_inited
) {
372 iommu_table_dart_inited
= 1;
373 iommu_table_dart_setup();
377 static bool dart_device_on_pcie(struct device
*dev
)
379 struct device_node
*np
= of_node_get(dev
->of_node
);
382 if (of_device_is_compatible(np
, "U4-pcie") ||
383 of_device_is_compatible(np
, "u4-pcie")) {
387 np
= of_get_next_parent(np
);
392 static void pci_dma_dev_setup_dart(struct pci_dev
*dev
)
394 if (dart_is_u4
&& dart_device_on_pcie(&dev
->dev
))
395 dev
->dev
.archdata
.dma_offset
= DART_U4_BYPASS_BASE
;
396 set_iommu_table_base(&dev
->dev
, &iommu_table_dart
);
399 static bool iommu_bypass_supported_dart(struct pci_dev
*dev
, u64 mask
)
402 dart_device_on_pcie(&dev
->dev
) &&
403 mask
>= DMA_BIT_MASK(40);
406 void __init
iommu_init_early_dart(struct pci_controller_ops
*controller_ops
)
408 struct device_node
*dn
;
410 /* Find the DART in the device-tree */
411 dn
= of_find_compatible_node(NULL
, "dart", "u3-dart");
413 dn
= of_find_compatible_node(NULL
, "dart", "u4-dart");
415 return; /* use default direct_dma_ops */
419 /* Initialize the DART HW */
420 if (dart_init(dn
) != 0)
424 * U4 supports a DART bypass, we use it for 64-bit capable devices to
425 * improve performance. However, that only works for devices connected
426 * to the U4 own PCIe interface, not bridged through hypertransport.
427 * We need the device to support at least 40 bits of addresses.
429 controller_ops
->dma_dev_setup
= pci_dma_dev_setup_dart
;
430 controller_ops
->dma_bus_setup
= pci_dma_bus_setup_dart
;
431 controller_ops
->iommu_bypass_supported
= iommu_bypass_supported_dart
;
433 /* Setup pci_dma ops */
434 set_pci_dma_ops(&dma_iommu_ops
);
438 static void iommu_dart_restore(void)
440 dart_cache_sync(dart_tablebase
, dart_tablesize
/ sizeof(u32
));
441 dart_tlb_invalidate_all();
444 static int __init
iommu_init_late_dart(void)
449 ppc_md
.iommu_restore
= iommu_dart_restore
;
454 late_initcall(iommu_init_late_dart
);
455 #endif /* CONFIG_PM */