]>
Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
1beb6a7d | 2 | * arch/powerpc/sysdev/dart_iommu.c |
1da177e4 | 3 | * |
91f14480 | 4 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
1beb6a7d BH |
5 | * Copyright (C) 2005 Benjamin Herrenschmidt <benh@kernel.crashing.org>, |
6 | * IBM Corporation | |
1da177e4 LT |
7 | * |
8 | * Based on pSeries_iommu.c: | |
9 | * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation | |
91f14480 | 10 | * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation |
1da177e4 | 11 | * |
1beb6a7d BH |
12 | * Dynamic DMA mapping support, Apple U3, U4 & IBM CPC925 "DART" iommu. |
13 | * | |
1da177e4 | 14 | * |
1da177e4 LT |
15 | * This program is free software; you can redistribute it and/or modify |
16 | * it under the terms of the GNU General Public License as published by | |
17 | * the Free Software Foundation; either version 2 of the License, or | |
18 | * (at your option) any later version. | |
1beb6a7d | 19 | * |
1da177e4 LT |
20 | * This program is distributed in the hope that it will be useful, |
21 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
22 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
23 | * GNU General Public License for more details. | |
1beb6a7d | 24 | * |
1da177e4 LT |
25 | * You should have received a copy of the GNU General Public License |
26 | * along with this program; if not, write to the Free Software | |
27 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
28 | */ | |
29 | ||
1da177e4 LT |
30 | #include <linux/init.h> |
31 | #include <linux/types.h> | |
1da177e4 LT |
32 | #include <linux/mm.h> |
33 | #include <linux/spinlock.h> | |
34 | #include <linux/string.h> | |
35 | #include <linux/pci.h> | |
36 | #include <linux/dma-mapping.h> | |
37 | #include <linux/vmalloc.h> | |
7e11580b | 38 | #include <linux/suspend.h> |
95f72d1e | 39 | #include <linux/memblock.h> |
5a0e3ad6 | 40 | #include <linux/gfp.h> |
1da177e4 LT |
41 | #include <asm/io.h> |
42 | #include <asm/prom.h> | |
1da177e4 LT |
43 | #include <asm/iommu.h> |
44 | #include <asm/pci-bridge.h> | |
45 | #include <asm/machdep.h> | |
46 | #include <asm/abs_addr.h> | |
47 | #include <asm/cacheflush.h> | |
d387899f | 48 | #include <asm/ppc-pci.h> |
1da177e4 | 49 | |
9933f299 DG |
50 | #include "dart.h" |
51 | ||
1da177e4 LT |
52 | /* Physical base address and size of the DART table */ |
53 | unsigned long dart_tablebase; /* exported to htab_initialize */ | |
54 | static unsigned long dart_tablesize; | |
55 | ||
56 | /* Virtual base address of the DART table */ | |
57 | static u32 *dart_vbase; | |
7e11580b JB |
58 | #ifdef CONFIG_PM |
59 | static u32 *dart_copy; | |
60 | #endif | |
1da177e4 LT |
61 | |
62 | /* Mapped base address for the dart */ | |
6fa2ffe9 | 63 | static unsigned int __iomem *dart; |
1da177e4 LT |
64 | |
65 | /* Dummy val that entries are set to when unused */ | |
66 | static unsigned int dart_emptyval; | |
67 | ||
1beb6a7d BH |
68 | static struct iommu_table iommu_table_dart; |
69 | static int iommu_table_dart_inited; | |
1da177e4 | 70 | static int dart_dirty; |
1beb6a7d | 71 | static int dart_is_u4; |
1da177e4 | 72 | |
8fb07c04 BH |
73 | #define DART_U4_BYPASS_BASE 0x8000000000ull |
74 | ||
1da177e4 LT |
75 | #define DBG(...) |
76 | ||
77 | static inline void dart_tlb_invalidate_all(void) | |
78 | { | |
79 | unsigned long l = 0; | |
1beb6a7d | 80 | unsigned int reg, inv_bit; |
1da177e4 LT |
81 | unsigned long limit; |
82 | ||
83 | DBG("dart: flush\n"); | |
84 | ||
85 | /* To invalidate the DART, set the DARTCNTL_FLUSHTLB bit in the | |
86 | * control register and wait for it to clear. | |
87 | * | |
88 | * Gotcha: Sometimes, the DART won't detect that the bit gets | |
89 | * set. If so, clear it and set it again. | |
1beb6a7d | 90 | */ |
1da177e4 LT |
91 | |
92 | limit = 0; | |
93 | ||
1beb6a7d | 94 | inv_bit = dart_is_u4 ? DART_CNTL_U4_FLUSHTLB : DART_CNTL_U3_FLUSHTLB; |
1da177e4 | 95 | retry: |
1da177e4 | 96 | l = 0; |
1beb6a7d BH |
97 | reg = DART_IN(DART_CNTL); |
98 | reg |= inv_bit; | |
99 | DART_OUT(DART_CNTL, reg); | |
100 | ||
101 | while ((DART_IN(DART_CNTL) & inv_bit) && l < (1L << limit)) | |
1da177e4 | 102 | l++; |
1beb6a7d | 103 | if (l == (1L << limit)) { |
1da177e4 LT |
104 | if (limit < 4) { |
105 | limit++; | |
feb76c7b OJ |
106 | reg = DART_IN(DART_CNTL); |
107 | reg &= ~inv_bit; | |
1beb6a7d | 108 | DART_OUT(DART_CNTL, reg); |
1da177e4 LT |
109 | goto retry; |
110 | } else | |
1beb6a7d | 111 | panic("DART: TLB did not flush after waiting a long " |
1da177e4 LT |
112 | "time. Buggy U3 ?"); |
113 | } | |
114 | } | |
115 | ||
feb76c7b OJ |
116 | static inline void dart_tlb_invalidate_one(unsigned long bus_rpn) |
117 | { | |
118 | unsigned int reg; | |
119 | unsigned int l, limit; | |
120 | ||
121 | reg = DART_CNTL_U4_ENABLE | DART_CNTL_U4_IONE | | |
122 | (bus_rpn & DART_CNTL_U4_IONE_MASK); | |
123 | DART_OUT(DART_CNTL, reg); | |
124 | ||
125 | limit = 0; | |
126 | wait_more: | |
127 | l = 0; | |
128 | while ((DART_IN(DART_CNTL) & DART_CNTL_U4_IONE) && l < (1L << limit)) { | |
129 | rmb(); | |
130 | l++; | |
131 | } | |
132 | ||
133 | if (l == (1L << limit)) { | |
134 | if (limit < 4) { | |
135 | limit++; | |
136 | goto wait_more; | |
137 | } else | |
138 | panic("DART: TLB did not flush after waiting a long " | |
139 | "time. Buggy U4 ?"); | |
140 | } | |
141 | } | |
142 | ||
1da177e4 LT |
143 | static void dart_flush(struct iommu_table *tbl) |
144 | { | |
eeac5c14 | 145 | mb(); |
feb76c7b | 146 | if (dart_dirty) { |
1da177e4 | 147 | dart_tlb_invalidate_all(); |
feb76c7b OJ |
148 | dart_dirty = 0; |
149 | } | |
1da177e4 LT |
150 | } |
151 | ||
6490c490 | 152 | static int dart_build(struct iommu_table *tbl, long index, |
1da177e4 | 153 | long npages, unsigned long uaddr, |
4f3dd8a0 MN |
154 | enum dma_data_direction direction, |
155 | struct dma_attrs *attrs) | |
1da177e4 LT |
156 | { |
157 | unsigned int *dp; | |
158 | unsigned int rpn; | |
feb76c7b | 159 | long l; |
1da177e4 LT |
160 | |
161 | DBG("dart: build at: %lx, %lx, addr: %x\n", index, npages, uaddr); | |
162 | ||
163 | dp = ((unsigned int*)tbl->it_base) + index; | |
1beb6a7d | 164 | |
af901ca1 | 165 | /* On U3, all memory is contiguous, so we can move this |
1da177e4 LT |
166 | * out of the loop. |
167 | */ | |
feb76c7b OJ |
168 | l = npages; |
169 | while (l--) { | |
d0035c62 | 170 | rpn = virt_to_abs(uaddr) >> DART_PAGE_SHIFT; |
1da177e4 LT |
171 | |
172 | *(dp++) = DARTMAP_VALID | (rpn & DARTMAP_RPNMASK); | |
173 | ||
d0035c62 | 174 | uaddr += DART_PAGE_SIZE; |
1da177e4 LT |
175 | } |
176 | ||
eeac5c14 BH |
177 | /* make sure all updates have reached memory */ |
178 | mb(); | |
179 | in_be32((unsigned __iomem *)dp); | |
180 | mb(); | |
181 | ||
feb76c7b OJ |
182 | if (dart_is_u4) { |
183 | rpn = index; | |
feb76c7b OJ |
184 | while (npages--) |
185 | dart_tlb_invalidate_one(rpn++); | |
186 | } else { | |
187 | dart_dirty = 1; | |
188 | } | |
6490c490 | 189 | return 0; |
1da177e4 LT |
190 | } |
191 | ||
192 | ||
193 | static void dart_free(struct iommu_table *tbl, long index, long npages) | |
194 | { | |
195 | unsigned int *dp; | |
1beb6a7d | 196 | |
1da177e4 LT |
197 | /* We don't worry about flushing the TLB cache. The only drawback of |
198 | * not doing it is that we won't catch buggy device drivers doing | |
199 | * bad DMAs, but then no 32-bit architecture ever does either. | |
200 | */ | |
201 | ||
202 | DBG("dart: free at: %lx, %lx\n", index, npages); | |
203 | ||
204 | dp = ((unsigned int *)tbl->it_base) + index; | |
1beb6a7d | 205 | |
1da177e4 LT |
206 | while (npages--) |
207 | *(dp++) = dart_emptyval; | |
208 | } | |
209 | ||
210 | ||
109b60f0 | 211 | static int __init dart_init(struct device_node *dart_node) |
1da177e4 | 212 | { |
1da177e4 | 213 | unsigned int i; |
1beb6a7d BH |
214 | unsigned long tmp, base, size; |
215 | struct resource r; | |
1da177e4 LT |
216 | |
217 | if (dart_tablebase == 0 || dart_tablesize == 0) { | |
1beb6a7d BH |
218 | printk(KERN_INFO "DART: table not allocated, using " |
219 | "direct DMA\n"); | |
1da177e4 LT |
220 | return -ENODEV; |
221 | } | |
222 | ||
1beb6a7d BH |
223 | if (of_address_to_resource(dart_node, 0, &r)) |
224 | panic("DART: can't get register base ! "); | |
225 | ||
1da177e4 LT |
226 | /* Make sure nothing from the DART range remains in the CPU cache |
227 | * from a previous mapping that existed before the kernel took | |
228 | * over | |
229 | */ | |
1beb6a7d BH |
230 | flush_dcache_phys_range(dart_tablebase, |
231 | dart_tablebase + dart_tablesize); | |
1da177e4 LT |
232 | |
233 | /* Allocate a spare page to map all invalid DART pages. We need to do | |
234 | * that to work around what looks like a problem with the HT bridge | |
235 | * prefetching into invalid pages and corrupting data | |
236 | */ | |
95f72d1e | 237 | tmp = memblock_alloc(DART_PAGE_SIZE, DART_PAGE_SIZE); |
1beb6a7d BH |
238 | dart_emptyval = DARTMAP_VALID | ((tmp >> DART_PAGE_SHIFT) & |
239 | DARTMAP_RPNMASK); | |
1da177e4 | 240 | |
1beb6a7d BH |
241 | /* Map in DART registers */ |
242 | dart = ioremap(r.start, r.end - r.start + 1); | |
1da177e4 | 243 | if (dart == NULL) |
1beb6a7d | 244 | panic("DART: Cannot map registers!"); |
1da177e4 | 245 | |
1beb6a7d | 246 | /* Map in DART table */ |
1da177e4 LT |
247 | dart_vbase = ioremap(virt_to_abs(dart_tablebase), dart_tablesize); |
248 | ||
249 | /* Fill initial table */ | |
250 | for (i = 0; i < dart_tablesize/4; i++) | |
251 | dart_vbase[i] = dart_emptyval; | |
252 | ||
253 | /* Initialize DART with table base and enable it. */ | |
1beb6a7d BH |
254 | base = dart_tablebase >> DART_PAGE_SHIFT; |
255 | size = dart_tablesize >> DART_PAGE_SHIFT; | |
256 | if (dart_is_u4) { | |
56c8eaee | 257 | size &= DART_SIZE_U4_SIZE_MASK; |
1beb6a7d BH |
258 | DART_OUT(DART_BASE_U4, base); |
259 | DART_OUT(DART_SIZE_U4, size); | |
260 | DART_OUT(DART_CNTL, DART_CNTL_U4_ENABLE); | |
261 | } else { | |
56c8eaee | 262 | size &= DART_CNTL_U3_SIZE_MASK; |
1beb6a7d BH |
263 | DART_OUT(DART_CNTL, |
264 | DART_CNTL_U3_ENABLE | | |
265 | (base << DART_CNTL_U3_BASE_SHIFT) | | |
266 | (size << DART_CNTL_U3_SIZE_SHIFT)); | |
267 | } | |
1da177e4 LT |
268 | |
269 | /* Invalidate DART to get rid of possible stale TLBs */ | |
270 | dart_tlb_invalidate_all(); | |
271 | ||
1beb6a7d BH |
272 | printk(KERN_INFO "DART IOMMU initialized for %s type chipset\n", |
273 | dart_is_u4 ? "U4" : "U3"); | |
1da177e4 LT |
274 | |
275 | return 0; | |
276 | } | |
277 | ||
1beb6a7d | 278 | static void iommu_table_dart_setup(void) |
1da177e4 | 279 | { |
1beb6a7d BH |
280 | iommu_table_dart.it_busno = 0; |
281 | iommu_table_dart.it_offset = 0; | |
1da177e4 | 282 | /* it_size is in number of entries */ |
5d2efba6 | 283 | iommu_table_dart.it_size = dart_tablesize / sizeof(u32); |
1da177e4 LT |
284 | |
285 | /* Initialize the common IOMMU code */ | |
1beb6a7d BH |
286 | iommu_table_dart.it_base = (unsigned long)dart_vbase; |
287 | iommu_table_dart.it_index = 0; | |
288 | iommu_table_dart.it_blocksize = 1; | |
ca1588e7 | 289 | iommu_init_table(&iommu_table_dart, -1); |
1da177e4 LT |
290 | |
291 | /* Reserve the last page of the DART to avoid possible prefetch | |
292 | * past the DART mapped area | |
293 | */ | |
1beb6a7d | 294 | set_bit(iommu_table_dart.it_size - 1, iommu_table_dart.it_map); |
1da177e4 LT |
295 | } |
296 | ||
8fb07c04 | 297 | static void dma_dev_setup_dart(struct device *dev) |
1da177e4 | 298 | { |
1da177e4 LT |
299 | /* We only have one iommu table on the mac for now, which makes |
300 | * things simple. Setup all PCI devices to point to this table | |
1da177e4 | 301 | */ |
8fb07c04 BH |
302 | if (get_dma_ops(dev) == &dma_direct_ops) |
303 | set_dma_offset(dev, DART_U4_BYPASS_BASE); | |
304 | else | |
305 | set_iommu_table_base(dev, &iommu_table_dart); | |
306 | } | |
307 | ||
308 | static void pci_dma_dev_setup_dart(struct pci_dev *dev) | |
309 | { | |
310 | dma_dev_setup_dart(&dev->dev); | |
1da177e4 LT |
311 | } |
312 | ||
12d04eef | 313 | static void pci_dma_bus_setup_dart(struct pci_bus *bus) |
1da177e4 | 314 | { |
1beb6a7d BH |
315 | if (!iommu_table_dart_inited) { |
316 | iommu_table_dart_inited = 1; | |
317 | iommu_table_dart_setup(); | |
1da177e4 | 318 | } |
1da177e4 LT |
319 | } |
320 | ||
8fb07c04 BH |
321 | static bool dart_device_on_pcie(struct device *dev) |
322 | { | |
323 | struct device_node *np = of_node_get(dev->of_node); | |
324 | ||
325 | while(np) { | |
326 | if (of_device_is_compatible(np, "U4-pcie") || | |
327 | of_device_is_compatible(np, "u4-pcie")) { | |
328 | of_node_put(np); | |
329 | return true; | |
330 | } | |
331 | np = of_get_next_parent(np); | |
332 | } | |
333 | return false; | |
334 | } | |
335 | ||
336 | static int dart_dma_set_mask(struct device *dev, u64 dma_mask) | |
337 | { | |
338 | if (!dev->dma_mask || !dma_supported(dev, dma_mask)) | |
339 | return -EIO; | |
340 | ||
341 | /* U4 supports a DART bypass, we use it for 64-bit capable | |
342 | * devices to improve performances. However, that only works | |
343 | * for devices connected to U4 own PCIe interface, not bridged | |
344 | * through hypertransport. We need the device to support at | |
345 | * least 40 bits of addresses. | |
346 | */ | |
347 | if (dart_device_on_pcie(dev) && dma_mask >= DMA_BIT_MASK(40)) { | |
348 | dev_info(dev, "Using 64-bit DMA iommu bypass\n"); | |
349 | set_dma_ops(dev, &dma_direct_ops); | |
350 | } else { | |
351 | dev_info(dev, "Using 32-bit DMA via iommu\n"); | |
352 | set_dma_ops(dev, &dma_iommu_ops); | |
353 | } | |
354 | dma_dev_setup_dart(dev); | |
355 | ||
356 | *dev->dma_mask = dma_mask; | |
357 | return 0; | |
358 | } | |
359 | ||
109b60f0 | 360 | void __init iommu_init_early_dart(void) |
1da177e4 LT |
361 | { |
362 | struct device_node *dn; | |
363 | ||
364 | /* Find the DART in the device-tree */ | |
365 | dn = of_find_compatible_node(NULL, "dart", "u3-dart"); | |
1beb6a7d BH |
366 | if (dn == NULL) { |
367 | dn = of_find_compatible_node(NULL, "dart", "u4-dart"); | |
368 | if (dn == NULL) | |
34c4d012 | 369 | return; /* use default direct_dma_ops */ |
1beb6a7d BH |
370 | dart_is_u4 = 1; |
371 | } | |
1da177e4 | 372 | |
8fb07c04 BH |
373 | /* Initialize the DART HW */ |
374 | if (dart_init(dn) != 0) | |
375 | goto bail; | |
376 | ||
1da177e4 LT |
377 | /* Setup low level TCE operations for the core IOMMU code */ |
378 | ppc_md.tce_build = dart_build; | |
379 | ppc_md.tce_free = dart_free; | |
380 | ppc_md.tce_flush = dart_flush; | |
381 | ||
8fb07c04 BH |
382 | /* Setup bypass if supported */ |
383 | if (dart_is_u4) | |
384 | ppc_md.dma_set_mask = dart_dma_set_mask; | |
1da177e4 | 385 | |
8fb07c04 BH |
386 | ppc_md.pci_dma_dev_setup = pci_dma_dev_setup_dart; |
387 | ppc_md.pci_dma_bus_setup = pci_dma_bus_setup_dart; | |
388 | ||
389 | /* Setup pci_dma ops */ | |
390 | set_pci_dma_ops(&dma_iommu_ops); | |
391 | return; | |
1beb6a7d BH |
392 | |
393 | bail: | |
394 | /* If init failed, use direct iommu and null setup functions */ | |
12d04eef BH |
395 | ppc_md.pci_dma_dev_setup = NULL; |
396 | ppc_md.pci_dma_bus_setup = NULL; | |
1beb6a7d BH |
397 | |
398 | /* Setup pci_dma ops */ | |
98747770 | 399 | set_pci_dma_ops(&dma_direct_ops); |
1da177e4 LT |
400 | } |
401 | ||
7e11580b JB |
402 | #ifdef CONFIG_PM |
403 | static void iommu_dart_save(void) | |
404 | { | |
405 | memcpy(dart_copy, dart_vbase, 2*1024*1024); | |
406 | } | |
407 | ||
408 | static void iommu_dart_restore(void) | |
409 | { | |
410 | memcpy(dart_vbase, dart_copy, 2*1024*1024); | |
411 | dart_tlb_invalidate_all(); | |
412 | } | |
413 | ||
414 | static int __init iommu_init_late_dart(void) | |
415 | { | |
416 | unsigned long tbasepfn; | |
417 | struct page *p; | |
418 | ||
419 | /* if no dart table exists then we won't need to save it | |
420 | * and the area has also not been reserved */ | |
421 | if (!dart_tablebase) | |
422 | return 0; | |
423 | ||
424 | tbasepfn = __pa(dart_tablebase) >> PAGE_SHIFT; | |
425 | register_nosave_region_late(tbasepfn, | |
426 | tbasepfn + ((1<<24) >> PAGE_SHIFT)); | |
427 | ||
428 | /* For suspend we need to copy the dart contents because | |
429 | * it is not part of the regular mapping (see above) and | |
430 | * thus not saved automatically. The memory for this copy | |
431 | * must be allocated early because we need 2 MB. */ | |
432 | p = alloc_pages(GFP_KERNEL, 21 - PAGE_SHIFT); | |
433 | BUG_ON(!p); | |
434 | dart_copy = page_address(p); | |
435 | ||
436 | ppc_md.iommu_save = iommu_dart_save; | |
437 | ppc_md.iommu_restore = iommu_dart_restore; | |
438 | ||
439 | return 0; | |
440 | } | |
441 | ||
442 | late_initcall(iommu_init_late_dart); | |
443 | #endif | |
1da177e4 | 444 | |
1beb6a7d | 445 | void __init alloc_dart_table(void) |
1da177e4 | 446 | { |
28897731 | 447 | /* Only reserve DART space if machine has more than 1GB of RAM |
1da177e4 | 448 | * or if requested with iommu=on on cmdline. |
28897731 OJ |
449 | * |
450 | * 1GB of RAM is picked as limit because some default devices | |
451 | * (i.e. Airport Extreme) have 30 bit address range limits. | |
1da177e4 | 452 | */ |
28897731 OJ |
453 | |
454 | if (iommu_is_off) | |
455 | return; | |
456 | ||
95f72d1e | 457 | if (!iommu_force_on && memblock_end_of_DRAM() <= 0x40000000ull) |
1da177e4 LT |
458 | return; |
459 | ||
460 | /* 512 pages (2MB) is max DART tablesize. */ | |
461 | dart_tablesize = 1UL << 21; | |
462 | /* 16MB (1 << 24) alignment. We allocate a full 16Mb chuck since we | |
463 | * will blow up an entire large page anyway in the kernel mapping | |
464 | */ | |
465 | dart_tablebase = (unsigned long) | |
95f72d1e | 466 | abs_to_virt(memblock_alloc_base(1UL<<24, 1UL<<24, 0x80000000L)); |
1da177e4 | 467 | |
1beb6a7d | 468 | printk(KERN_INFO "DART table allocated at: %lx\n", dart_tablebase); |
1da177e4 | 469 | } |