]> git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blame - arch/powerpc/kernel/iommu.c
powerpc: Update currituck pci/usb fixup for new board revision
[mirror_ubuntu-zesty-kernel.git] / arch / powerpc / kernel / iommu.c
CommitLineData
1da177e4 1/*
1da177e4
LT
2 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
3 *
4 * Rewrite, cleanup, new allocation schemes, virtual merging:
5 * Copyright (C) 2004 Olof Johansson, IBM Corporation
6 * and Ben. Herrenschmidt, IBM Corporation
7 *
8 * Dynamic DMA mapping support, bus-independent parts.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 */
24
25
1da177e4
LT
26#include <linux/init.h>
27#include <linux/types.h>
28#include <linux/slab.h>
29#include <linux/mm.h>
30#include <linux/spinlock.h>
31#include <linux/string.h>
32#include <linux/dma-mapping.h>
a66022c4 33#include <linux/bitmap.h>
fb3475e9 34#include <linux/iommu-helper.h>
62a8bd6c 35#include <linux/crash_dump.h>
b4c3a872 36#include <linux/hash.h>
d6b9a81b
AB
37#include <linux/fault-inject.h>
38#include <linux/pci.h>
1da177e4
LT
39#include <asm/io.h>
40#include <asm/prom.h>
41#include <asm/iommu.h>
42#include <asm/pci-bridge.h>
43#include <asm/machdep.h>
5f50867b 44#include <asm/kdump.h>
3ccc00a7 45#include <asm/fadump.h>
d6b9a81b 46#include <asm/vio.h>
1da177e4
LT
47
48#define DBG(...)
49
191aee58 50static int novmerge;
56997559 51
6490c490
RJ
52static void __iommu_free(struct iommu_table *, dma_addr_t, unsigned int);
53
1da177e4
LT
54static int __init setup_iommu(char *str)
55{
56 if (!strcmp(str, "novmerge"))
57 novmerge = 1;
58 else if (!strcmp(str, "vmerge"))
59 novmerge = 0;
60 return 1;
61}
62
63__setup("iommu=", setup_iommu);
64
b4c3a872
AB
65static DEFINE_PER_CPU(unsigned int, iommu_pool_hash);
66
67/*
68 * We precalculate the hash to avoid doing it on every allocation.
69 *
70 * The hash is important to spread CPUs across all the pools. For example,
71 * on a POWER7 with 4 way SMT we want interrupts on the primary threads and
72 * with 4 pools all primary threads would map to the same pool.
73 */
74static int __init setup_iommu_pool_hash(void)
75{
76 unsigned int i;
77
78 for_each_possible_cpu(i)
79 per_cpu(iommu_pool_hash, i) = hash_32(i, IOMMU_POOL_HASHBITS);
80
81 return 0;
82}
83subsys_initcall(setup_iommu_pool_hash);
84
d6b9a81b
AB
85#ifdef CONFIG_FAIL_IOMMU
86
87static DECLARE_FAULT_ATTR(fail_iommu);
88
89static int __init setup_fail_iommu(char *str)
90{
91 return setup_fault_attr(&fail_iommu, str);
92}
93__setup("fail_iommu=", setup_fail_iommu);
94
95static bool should_fail_iommu(struct device *dev)
96{
97 return dev->archdata.fail_iommu && should_fail(&fail_iommu, 1);
98}
99
100static int __init fail_iommu_debugfs(void)
101{
102 struct dentry *dir = fault_create_debugfs_attr("fail_iommu",
103 NULL, &fail_iommu);
104
09652b00 105 return PTR_RET(dir);
d6b9a81b
AB
106}
107late_initcall(fail_iommu_debugfs);
108
109static ssize_t fail_iommu_show(struct device *dev,
110 struct device_attribute *attr, char *buf)
111{
112 return sprintf(buf, "%d\n", dev->archdata.fail_iommu);
113}
114
115static ssize_t fail_iommu_store(struct device *dev,
116 struct device_attribute *attr, const char *buf,
117 size_t count)
118{
119 int i;
120
121 if (count > 0 && sscanf(buf, "%d", &i) > 0)
122 dev->archdata.fail_iommu = (i == 0) ? 0 : 1;
123
124 return count;
125}
126
127static DEVICE_ATTR(fail_iommu, S_IRUGO|S_IWUSR, fail_iommu_show,
128 fail_iommu_store);
129
130static int fail_iommu_bus_notify(struct notifier_block *nb,
131 unsigned long action, void *data)
132{
133 struct device *dev = data;
134
135 if (action == BUS_NOTIFY_ADD_DEVICE) {
136 if (device_create_file(dev, &dev_attr_fail_iommu))
137 pr_warn("Unable to create IOMMU fault injection sysfs "
138 "entries\n");
139 } else if (action == BUS_NOTIFY_DEL_DEVICE) {
140 device_remove_file(dev, &dev_attr_fail_iommu);
141 }
142
143 return 0;
144}
145
146static struct notifier_block fail_iommu_bus_notifier = {
147 .notifier_call = fail_iommu_bus_notify
148};
149
150static int __init fail_iommu_setup(void)
151{
152#ifdef CONFIG_PCI
153 bus_register_notifier(&pci_bus_type, &fail_iommu_bus_notifier);
154#endif
155#ifdef CONFIG_IBMVIO
156 bus_register_notifier(&vio_bus_type, &fail_iommu_bus_notifier);
157#endif
158
159 return 0;
160}
161/*
162 * Must execute after PCI and VIO subsystem have initialised but before
163 * devices are probed.
164 */
165arch_initcall(fail_iommu_setup);
166#else
167static inline bool should_fail_iommu(struct device *dev)
168{
169 return false;
170}
171#endif
172
fb3475e9
FT
173static unsigned long iommu_range_alloc(struct device *dev,
174 struct iommu_table *tbl,
1da177e4
LT
175 unsigned long npages,
176 unsigned long *handle,
7daa411b 177 unsigned long mask,
1da177e4
LT
178 unsigned int align_order)
179{
fb3475e9 180 unsigned long n, end, start;
1da177e4
LT
181 unsigned long limit;
182 int largealloc = npages > 15;
183 int pass = 0;
184 unsigned long align_mask;
fb3475e9 185 unsigned long boundary_size;
d3622137 186 unsigned long flags;
b4c3a872
AB
187 unsigned int pool_nr;
188 struct iommu_pool *pool;
1da177e4
LT
189
190 align_mask = 0xffffffffffffffffl >> (64 - align_order);
191
192 /* This allocator was derived from x86_64's bit string search */
193
194 /* Sanity check */
13a2eea1 195 if (unlikely(npages == 0)) {
1da177e4
LT
196 if (printk_ratelimit())
197 WARN_ON(1);
198 return DMA_ERROR_CODE;
199 }
200
d6b9a81b
AB
201 if (should_fail_iommu(dev))
202 return DMA_ERROR_CODE;
203
b4c3a872
AB
204 /*
205 * We don't need to disable preemption here because any CPU can
206 * safely use any IOMMU pool.
207 */
208 pool_nr = __raw_get_cpu_var(iommu_pool_hash) & (tbl->nr_pools - 1);
d3622137 209
b4c3a872
AB
210 if (largealloc)
211 pool = &(tbl->large_pool);
1da177e4 212 else
b4c3a872 213 pool = &(tbl->pools[pool_nr]);
1da177e4 214
b4c3a872
AB
215 spin_lock_irqsave(&(pool->lock), flags);
216
217again:
d900bd73
AB
218 if ((pass == 0) && handle && *handle &&
219 (*handle >= pool->start) && (*handle < pool->end))
b4c3a872
AB
220 start = *handle;
221 else
222 start = pool->hint;
1da177e4 223
b4c3a872 224 limit = pool->end;
1da177e4
LT
225
226 /* The case below can happen if we have a small segment appended
227 * to a large, or when the previous alloc was at the very end of
228 * the available space. If so, go back to the initial start.
229 */
230 if (start >= limit)
b4c3a872 231 start = pool->start;
1da177e4 232
7daa411b
OJ
233 if (limit + tbl->it_offset > mask) {
234 limit = mask - tbl->it_offset + 1;
235 /* If we're constrained on address range, first try
236 * at the masked hint to avoid O(n) search complexity,
b4c3a872 237 * but on second pass, start at 0 in pool 0.
7daa411b 238 */
b4c3a872 239 if ((start & mask) >= limit || pass > 0) {
d900bd73 240 spin_unlock(&(pool->lock));
b4c3a872 241 pool = &(tbl->pools[0]);
d900bd73 242 spin_lock(&(pool->lock));
b4c3a872
AB
243 start = pool->start;
244 } else {
7daa411b 245 start &= mask;
b4c3a872 246 }
7daa411b
OJ
247 }
248
fb3475e9
FT
249 if (dev)
250 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1,
251 1 << IOMMU_PAGE_SHIFT);
252 else
253 boundary_size = ALIGN(1UL << 32, 1 << IOMMU_PAGE_SHIFT);
254 /* 4GB boundary for iseries_hv_alloc and iseries_hv_map */
1da177e4 255
fb3475e9
FT
256 n = iommu_area_alloc(tbl->it_map, limit, start, npages,
257 tbl->it_offset, boundary_size >> IOMMU_PAGE_SHIFT,
258 align_mask);
259 if (n == -1) {
b4c3a872
AB
260 if (likely(pass == 0)) {
261 /* First try the pool from the start */
262 pool->hint = pool->start;
1da177e4
LT
263 pass++;
264 goto again;
b4c3a872
AB
265
266 } else if (pass <= tbl->nr_pools) {
267 /* Now try scanning all the other pools */
268 spin_unlock(&(pool->lock));
269 pool_nr = (pool_nr + 1) & (tbl->nr_pools - 1);
270 pool = &tbl->pools[pool_nr];
271 spin_lock(&(pool->lock));
272 pool->hint = pool->start;
273 pass++;
274 goto again;
275
1da177e4 276 } else {
b4c3a872
AB
277 /* Give up */
278 spin_unlock_irqrestore(&(pool->lock), flags);
1da177e4
LT
279 return DMA_ERROR_CODE;
280 }
281 }
282
fb3475e9 283 end = n + npages;
1da177e4
LT
284
285 /* Bump the hint to a new block for small allocs. */
286 if (largealloc) {
287 /* Don't bump to new block to avoid fragmentation */
b4c3a872 288 pool->hint = end;
1da177e4
LT
289 } else {
290 /* Overflow will be taken care of at the next allocation */
b4c3a872 291 pool->hint = (end + tbl->it_blocksize - 1) &
1da177e4
LT
292 ~(tbl->it_blocksize - 1);
293 }
294
295 /* Update handle for SG allocations */
296 if (handle)
297 *handle = end;
298
b4c3a872
AB
299 spin_unlock_irqrestore(&(pool->lock), flags);
300
1da177e4
LT
301 return n;
302}
303
fb3475e9
FT
304static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl,
305 void *page, unsigned int npages,
306 enum dma_data_direction direction,
4f3dd8a0
MN
307 unsigned long mask, unsigned int align_order,
308 struct dma_attrs *attrs)
1da177e4 309{
d3622137 310 unsigned long entry;
1da177e4 311 dma_addr_t ret = DMA_ERROR_CODE;
6490c490 312 int build_fail;
7daa411b 313
fb3475e9 314 entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order);
1da177e4 315
0e4bc95d 316 if (unlikely(entry == DMA_ERROR_CODE))
1da177e4 317 return DMA_ERROR_CODE;
1da177e4
LT
318
319 entry += tbl->it_offset; /* Offset into real TCE table */
5d2efba6 320 ret = entry << IOMMU_PAGE_SHIFT; /* Set the return dma address */
1da177e4
LT
321
322 /* Put the TCEs in the HW table */
6490c490
RJ
323 build_fail = ppc_md.tce_build(tbl, entry, npages,
324 (unsigned long)page & IOMMU_PAGE_MASK,
325 direction, attrs);
326
327 /* ppc_md.tce_build() only returns non-zero for transient errors.
328 * Clean up the table bitmap in this case and return
329 * DMA_ERROR_CODE. For all other errors the functionality is
330 * not altered.
331 */
332 if (unlikely(build_fail)) {
333 __iommu_free(tbl, ret, npages);
6490c490
RJ
334 return DMA_ERROR_CODE;
335 }
1da177e4
LT
336
337 /* Flush/invalidate TLB caches if necessary */
338 if (ppc_md.tce_flush)
339 ppc_md.tce_flush(tbl);
340
1da177e4
LT
341 /* Make sure updates are seen by hardware */
342 mb();
343
344 return ret;
345}
346
67ca1415
AB
347static bool iommu_free_check(struct iommu_table *tbl, dma_addr_t dma_addr,
348 unsigned int npages)
1da177e4
LT
349{
350 unsigned long entry, free_entry;
1da177e4 351
5d2efba6 352 entry = dma_addr >> IOMMU_PAGE_SHIFT;
1da177e4
LT
353 free_entry = entry - tbl->it_offset;
354
355 if (((free_entry + npages) > tbl->it_size) ||
356 (entry < tbl->it_offset)) {
357 if (printk_ratelimit()) {
358 printk(KERN_INFO "iommu_free: invalid entry\n");
359 printk(KERN_INFO "\tentry = 0x%lx\n", entry);
fe333321
IM
360 printk(KERN_INFO "\tdma_addr = 0x%llx\n", (u64)dma_addr);
361 printk(KERN_INFO "\tTable = 0x%llx\n", (u64)tbl);
362 printk(KERN_INFO "\tbus# = 0x%llx\n", (u64)tbl->it_busno);
363 printk(KERN_INFO "\tsize = 0x%llx\n", (u64)tbl->it_size);
364 printk(KERN_INFO "\tstartOff = 0x%llx\n", (u64)tbl->it_offset);
365 printk(KERN_INFO "\tindex = 0x%llx\n", (u64)tbl->it_index);
1da177e4
LT
366 WARN_ON(1);
367 }
67ca1415
AB
368
369 return false;
1da177e4
LT
370 }
371
67ca1415
AB
372 return true;
373}
374
b4c3a872
AB
375static struct iommu_pool *get_pool(struct iommu_table *tbl,
376 unsigned long entry)
377{
378 struct iommu_pool *p;
379 unsigned long largepool_start = tbl->large_pool.start;
380
381 /* The large pool is the last pool at the top of the table */
382 if (entry >= largepool_start) {
383 p = &tbl->large_pool;
384 } else {
385 unsigned int pool_nr = entry / tbl->poolsize;
386
387 BUG_ON(pool_nr > tbl->nr_pools);
388 p = &tbl->pools[pool_nr];
389 }
390
391 return p;
392}
393
67ca1415
AB
394static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
395 unsigned int npages)
1da177e4 396{
67ca1415 397 unsigned long entry, free_entry;
1da177e4 398 unsigned long flags;
b4c3a872 399 struct iommu_pool *pool;
1da177e4 400
67ca1415
AB
401 entry = dma_addr >> IOMMU_PAGE_SHIFT;
402 free_entry = entry - tbl->it_offset;
403
b4c3a872
AB
404 pool = get_pool(tbl, free_entry);
405
67ca1415
AB
406 if (!iommu_free_check(tbl, dma_addr, npages))
407 return;
408
409 ppc_md.tce_free(tbl, entry, npages);
410
b4c3a872 411 spin_lock_irqsave(&(pool->lock), flags);
67ca1415 412 bitmap_clear(tbl->it_map, free_entry, npages);
b4c3a872 413 spin_unlock_irqrestore(&(pool->lock), flags);
67ca1415
AB
414}
415
416static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr,
417 unsigned int npages)
418{
419 __iommu_free(tbl, dma_addr, npages);
1da177e4
LT
420
421 /* Make sure TLB cache is flushed if the HW needs it. We do
422 * not do an mb() here on purpose, it is not needed on any of
423 * the current platforms.
424 */
425 if (ppc_md.tce_flush)
426 ppc_md.tce_flush(tbl);
1da177e4
LT
427}
428
c8692362
MN
429int iommu_map_sg(struct device *dev, struct iommu_table *tbl,
430 struct scatterlist *sglist, int nelems,
3affedc4
MN
431 unsigned long mask, enum dma_data_direction direction,
432 struct dma_attrs *attrs)
1da177e4
LT
433{
434 dma_addr_t dma_next = 0, dma_addr;
1da177e4 435 struct scatterlist *s, *outs, *segstart;
6490c490 436 int outcount, incount, i, build_fail = 0;
d262c32a 437 unsigned int align;
1da177e4 438 unsigned long handle;
740c3ce6 439 unsigned int max_seg_size;
1da177e4
LT
440
441 BUG_ON(direction == DMA_NONE);
442
443 if ((nelems == 0) || !tbl)
444 return 0;
445
446 outs = s = segstart = &sglist[0];
447 outcount = 1;
ac9af7cb 448 incount = nelems;
1da177e4
LT
449 handle = 0;
450
451 /* Init first segment length for backout at failure */
452 outs->dma_length = 0;
453
5d2efba6 454 DBG("sg mapping %d elements:\n", nelems);
1da177e4 455
740c3ce6 456 max_seg_size = dma_get_max_seg_size(dev);
78bdc310 457 for_each_sg(sglist, s, nelems, i) {
1da177e4
LT
458 unsigned long vaddr, npages, entry, slen;
459
460 slen = s->length;
461 /* Sanity check */
462 if (slen == 0) {
463 dma_next = 0;
464 continue;
465 }
466 /* Allocate iommu entries for that segment */
58b053e4 467 vaddr = (unsigned long) sg_virt(s);
2994a3b2 468 npages = iommu_num_pages(vaddr, slen, IOMMU_PAGE_SIZE);
d262c32a
BH
469 align = 0;
470 if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && slen >= PAGE_SIZE &&
471 (vaddr & ~PAGE_MASK) == 0)
472 align = PAGE_SHIFT - IOMMU_PAGE_SHIFT;
fb3475e9 473 entry = iommu_range_alloc(dev, tbl, npages, &handle,
d262c32a 474 mask >> IOMMU_PAGE_SHIFT, align);
1da177e4
LT
475
476 DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen);
477
478 /* Handle failure */
479 if (unlikely(entry == DMA_ERROR_CODE)) {
480 if (printk_ratelimit())
4dfa9c47
AB
481 dev_info(dev, "iommu_alloc failed, tbl %p "
482 "vaddr %lx npages %lu\n", tbl, vaddr,
483 npages);
1da177e4
LT
484 goto failure;
485 }
486
487 /* Convert entry to a dma_addr_t */
488 entry += tbl->it_offset;
5d2efba6
LV
489 dma_addr = entry << IOMMU_PAGE_SHIFT;
490 dma_addr |= (s->offset & ~IOMMU_PAGE_MASK);
1da177e4 491
5d2efba6 492 DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n",
1da177e4
LT
493 npages, entry, dma_addr);
494
495 /* Insert into HW table */
6490c490
RJ
496 build_fail = ppc_md.tce_build(tbl, entry, npages,
497 vaddr & IOMMU_PAGE_MASK,
498 direction, attrs);
499 if(unlikely(build_fail))
500 goto failure;
1da177e4
LT
501
502 /* If we are in an open segment, try merging */
503 if (segstart != s) {
504 DBG(" - trying merge...\n");
505 /* We cannot merge if:
506 * - allocated dma_addr isn't contiguous to previous allocation
507 */
740c3ce6
FT
508 if (novmerge || (dma_addr != dma_next) ||
509 (outs->dma_length + s->length > max_seg_size)) {
1da177e4
LT
510 /* Can't merge: create a new segment */
511 segstart = s;
78bdc310
JA
512 outcount++;
513 outs = sg_next(outs);
1da177e4
LT
514 DBG(" can't merge, new segment.\n");
515 } else {
516 outs->dma_length += s->length;
5d2efba6 517 DBG(" merged, new len: %ux\n", outs->dma_length);
1da177e4
LT
518 }
519 }
520
521 if (segstart == s) {
522 /* This is a new segment, fill entries */
523 DBG(" - filling new segment.\n");
524 outs->dma_address = dma_addr;
525 outs->dma_length = slen;
526 }
527
528 /* Calculate next page pointer for contiguous check */
529 dma_next = dma_addr + slen;
530
531 DBG(" - dma next is: %lx\n", dma_next);
532 }
533
534 /* Flush/invalidate TLB caches if necessary */
535 if (ppc_md.tce_flush)
536 ppc_md.tce_flush(tbl);
537
1da177e4
LT
538 DBG("mapped %d elements:\n", outcount);
539
ac9af7cb 540 /* For the sake of iommu_unmap_sg, we clear out the length in the
1da177e4
LT
541 * next entry of the sglist if we didn't fill the list completely
542 */
ac9af7cb 543 if (outcount < incount) {
78bdc310 544 outs = sg_next(outs);
1da177e4
LT
545 outs->dma_address = DMA_ERROR_CODE;
546 outs->dma_length = 0;
547 }
a958a264
JM
548
549 /* Make sure updates are seen by hardware */
550 mb();
551
1da177e4
LT
552 return outcount;
553
554 failure:
78bdc310 555 for_each_sg(sglist, s, nelems, i) {
1da177e4
LT
556 if (s->dma_length != 0) {
557 unsigned long vaddr, npages;
558
5d2efba6 559 vaddr = s->dma_address & IOMMU_PAGE_MASK;
2994a3b2
JR
560 npages = iommu_num_pages(s->dma_address, s->dma_length,
561 IOMMU_PAGE_SIZE);
d3622137 562 __iommu_free(tbl, vaddr, npages);
a958a264
JM
563 s->dma_address = DMA_ERROR_CODE;
564 s->dma_length = 0;
1da177e4 565 }
78bdc310
JA
566 if (s == outs)
567 break;
1da177e4 568 }
1da177e4
LT
569 return 0;
570}
571
572
573void iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist,
3affedc4
MN
574 int nelems, enum dma_data_direction direction,
575 struct dma_attrs *attrs)
1da177e4 576{
78bdc310 577 struct scatterlist *sg;
1da177e4
LT
578
579 BUG_ON(direction == DMA_NONE);
580
581 if (!tbl)
582 return;
583
78bdc310 584 sg = sglist;
1da177e4
LT
585 while (nelems--) {
586 unsigned int npages;
78bdc310 587 dma_addr_t dma_handle = sg->dma_address;
1da177e4 588
78bdc310 589 if (sg->dma_length == 0)
1da177e4 590 break;
2994a3b2
JR
591 npages = iommu_num_pages(dma_handle, sg->dma_length,
592 IOMMU_PAGE_SIZE);
d3622137 593 __iommu_free(tbl, dma_handle, npages);
78bdc310 594 sg = sg_next(sg);
1da177e4
LT
595 }
596
597 /* Flush/invalidate TLBs if necessary. As for iommu_free(), we
598 * do not do an mb() here, the affected platforms do not need it
599 * when freeing.
600 */
601 if (ppc_md.tce_flush)
602 ppc_md.tce_flush(tbl);
1da177e4
LT
603}
604
54622f10
MK
605static void iommu_table_clear(struct iommu_table *tbl)
606{
3ccc00a7
MS
607 /*
608 * In case of firmware assisted dump system goes through clean
609 * reboot process at the time of system crash. Hence it's safe to
610 * clear the TCE entries if firmware assisted dump is active.
611 */
612 if (!is_kdump_kernel() || is_fadump_active()) {
54622f10
MK
613 /* Clear the table in case firmware left allocations in it */
614 ppc_md.tce_free(tbl, tbl->it_offset, tbl->it_size);
615 return;
616 }
617
618#ifdef CONFIG_CRASH_DUMP
619 if (ppc_md.tce_get) {
620 unsigned long index, tceval, tcecount = 0;
621
622 /* Reserve the existing mappings left by the first kernel. */
623 for (index = 0; index < tbl->it_size; index++) {
624 tceval = ppc_md.tce_get(tbl, index + tbl->it_offset);
625 /*
626 * Freed TCE entry contains 0x7fffffffffffffff on JS20
627 */
628 if (tceval && (tceval != 0x7fffffffffffffffUL)) {
629 __set_bit(index, tbl->it_map);
630 tcecount++;
631 }
632 }
633
634 if ((tbl->it_size - tcecount) < KDUMP_MIN_TCE_ENTRIES) {
635 printk(KERN_WARNING "TCE table is full; freeing ");
636 printk(KERN_WARNING "%d entries for the kdump boot\n",
637 KDUMP_MIN_TCE_ENTRIES);
638 for (index = tbl->it_size - KDUMP_MIN_TCE_ENTRIES;
639 index < tbl->it_size; index++)
640 __clear_bit(index, tbl->it_map);
641 }
642 }
643#endif
644}
645
1da177e4
LT
646/*
647 * Build a iommu_table structure. This contains a bit map which
648 * is used to manage allocation of the tce space.
649 */
ca1588e7 650struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid)
1da177e4
LT
651{
652 unsigned long sz;
653 static int welcomed = 0;
ca1588e7 654 struct page *page;
b4c3a872
AB
655 unsigned int i;
656 struct iommu_pool *p;
1da177e4
LT
657
658 /* number of bytes needed for the bitmap */
c5a0809a 659 sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
1da177e4 660
ca1588e7
AB
661 page = alloc_pages_node(nid, GFP_ATOMIC, get_order(sz));
662 if (!page)
1da177e4 663 panic("iommu_init_table: Can't allocate %ld bytes\n", sz);
ca1588e7 664 tbl->it_map = page_address(page);
1da177e4
LT
665 memset(tbl->it_map, 0, sz);
666
d12b524f
TLSC
667 /*
668 * Reserve page 0 so it will not be used for any mappings.
669 * This avoids buggy drivers that consider page 0 to be invalid
670 * to crash the machine or even lose data.
671 */
672 if (tbl->it_offset == 0)
673 set_bit(0, tbl->it_map);
674
b4c3a872
AB
675 /* We only split the IOMMU table if we have 1GB or more of space */
676 if ((tbl->it_size << IOMMU_PAGE_SHIFT) >= (1UL * 1024 * 1024 * 1024))
677 tbl->nr_pools = IOMMU_NR_POOLS;
678 else
679 tbl->nr_pools = 1;
680
681 /* We reserve the top 1/4 of the table for large allocations */
dcd261ba 682 tbl->poolsize = (tbl->it_size * 3 / 4) / tbl->nr_pools;
b4c3a872 683
dcd261ba 684 for (i = 0; i < tbl->nr_pools; i++) {
b4c3a872
AB
685 p = &tbl->pools[i];
686 spin_lock_init(&(p->lock));
687 p->start = tbl->poolsize * i;
688 p->hint = p->start;
689 p->end = p->start + tbl->poolsize;
690 }
691
692 p = &tbl->large_pool;
693 spin_lock_init(&(p->lock));
694 p->start = tbl->poolsize * i;
695 p->hint = p->start;
696 p->end = tbl->it_size;
1da177e4 697
54622f10 698 iommu_table_clear(tbl);
d3588ba9 699
1da177e4
LT
700 if (!welcomed) {
701 printk(KERN_INFO "IOMMU table initialized, virtual merging %s\n",
702 novmerge ? "disabled" : "enabled");
703 welcomed = 1;
704 }
705
706 return tbl;
707}
708
68d315f5 709void iommu_free_table(struct iommu_table *tbl, const char *node_name)
1da177e4 710{
c5a0809a 711 unsigned long bitmap_sz;
1da177e4
LT
712 unsigned int order;
713
714 if (!tbl || !tbl->it_map) {
e48b1b45 715 printk(KERN_ERR "%s: expected TCE map for %s\n", __func__,
68d315f5 716 node_name);
1da177e4
LT
717 return;
718 }
719
7f966d39
TLSC
720 /*
721 * In case we have reserved the first bit, we should not emit
722 * the warning below.
723 */
724 if (tbl->it_offset == 0)
725 clear_bit(0, tbl->it_map);
726
1da177e4 727 /* verify that table contains no entries */
c5a0809a
AM
728 if (!bitmap_empty(tbl->it_map, tbl->it_size))
729 pr_warn("%s: Unexpected TCEs for %s\n", __func__, node_name);
1da177e4
LT
730
731 /* calculate bitmap size in bytes */
c5a0809a 732 bitmap_sz = BITS_TO_LONGS(tbl->it_size) * sizeof(unsigned long);
1da177e4
LT
733
734 /* free bitmap */
735 order = get_order(bitmap_sz);
736 free_pages((unsigned long) tbl->it_map, order);
737
738 /* free table */
739 kfree(tbl);
740}
741
742/* Creates TCEs for a user provided buffer. The user buffer must be
f9226d57
MN
743 * contiguous real kernel storage (not vmalloc). The address passed here
744 * comprises a page address and offset into that page. The dma_addr_t
745 * returned will point to the same byte within the page as was passed in.
1da177e4 746 */
f9226d57
MN
747dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl,
748 struct page *page, unsigned long offset, size_t size,
749 unsigned long mask, enum dma_data_direction direction,
750 struct dma_attrs *attrs)
1da177e4
LT
751{
752 dma_addr_t dma_handle = DMA_ERROR_CODE;
f9226d57 753 void *vaddr;
1da177e4 754 unsigned long uaddr;
d262c32a 755 unsigned int npages, align;
1da177e4
LT
756
757 BUG_ON(direction == DMA_NONE);
758
f9226d57 759 vaddr = page_address(page) + offset;
1da177e4 760 uaddr = (unsigned long)vaddr;
2994a3b2 761 npages = iommu_num_pages(uaddr, size, IOMMU_PAGE_SIZE);
1da177e4
LT
762
763 if (tbl) {
d262c32a
BH
764 align = 0;
765 if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && size >= PAGE_SIZE &&
766 ((unsigned long)vaddr & ~PAGE_MASK) == 0)
767 align = PAGE_SHIFT - IOMMU_PAGE_SHIFT;
768
fb3475e9 769 dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction,
4f3dd8a0
MN
770 mask >> IOMMU_PAGE_SHIFT, align,
771 attrs);
1da177e4
LT
772 if (dma_handle == DMA_ERROR_CODE) {
773 if (printk_ratelimit()) {
4dfa9c47
AB
774 dev_info(dev, "iommu_alloc failed, tbl %p "
775 "vaddr %p npages %d\n", tbl, vaddr,
776 npages);
1da177e4
LT
777 }
778 } else
5d2efba6 779 dma_handle |= (uaddr & ~IOMMU_PAGE_MASK);
1da177e4
LT
780 }
781
782 return dma_handle;
783}
784
f9226d57
MN
785void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle,
786 size_t size, enum dma_data_direction direction,
787 struct dma_attrs *attrs)
1da177e4 788{
5d2efba6
LV
789 unsigned int npages;
790
1da177e4
LT
791 BUG_ON(direction == DMA_NONE);
792
5d2efba6 793 if (tbl) {
2994a3b2 794 npages = iommu_num_pages(dma_handle, size, IOMMU_PAGE_SIZE);
5d2efba6
LV
795 iommu_free(tbl, dma_handle, npages);
796 }
1da177e4
LT
797}
798
799/* Allocates a contiguous real buffer and creates mappings over it.
800 * Returns the virtual address of the buffer and sets dma_handle
801 * to the dma address (mapping) of the first page.
802 */
fb3475e9
FT
803void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl,
804 size_t size, dma_addr_t *dma_handle,
805 unsigned long mask, gfp_t flag, int node)
1da177e4
LT
806{
807 void *ret = NULL;
808 dma_addr_t mapping;
5d2efba6
LV
809 unsigned int order;
810 unsigned int nio_pages, io_order;
8eb6c6e3 811 struct page *page;
1da177e4
LT
812
813 size = PAGE_ALIGN(size);
1da177e4
LT
814 order = get_order(size);
815
816 /*
817 * Client asked for way too much space. This is checked later
818 * anyway. It is easier to debug here for the drivers than in
819 * the tce tables.
820 */
821 if (order >= IOMAP_MAX_ORDER) {
4dfa9c47
AB
822 dev_info(dev, "iommu_alloc_consistent size too large: 0x%lx\n",
823 size);
1da177e4
LT
824 return NULL;
825 }
826
827 if (!tbl)
828 return NULL;
829
830 /* Alloc enough pages (and possibly more) */
05061354 831 page = alloc_pages_node(node, flag, order);
8eb6c6e3 832 if (!page)
1da177e4 833 return NULL;
8eb6c6e3 834 ret = page_address(page);
1da177e4
LT
835 memset(ret, 0, size);
836
837 /* Set up tces to cover the allocated range */
5d2efba6
LV
838 nio_pages = size >> IOMMU_PAGE_SHIFT;
839 io_order = get_iommu_order(size);
fb3475e9 840 mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL,
4f3dd8a0 841 mask >> IOMMU_PAGE_SHIFT, io_order, NULL);
1da177e4
LT
842 if (mapping == DMA_ERROR_CODE) {
843 free_pages((unsigned long)ret, order);
8eb6c6e3
CH
844 return NULL;
845 }
846 *dma_handle = mapping;
1da177e4
LT
847 return ret;
848}
849
850void iommu_free_coherent(struct iommu_table *tbl, size_t size,
851 void *vaddr, dma_addr_t dma_handle)
852{
1da177e4 853 if (tbl) {
5d2efba6
LV
854 unsigned int nio_pages;
855
856 size = PAGE_ALIGN(size);
857 nio_pages = size >> IOMMU_PAGE_SHIFT;
858 iommu_free(tbl, dma_handle, nio_pages);
1da177e4 859 size = PAGE_ALIGN(size);
1da177e4
LT
860 free_pages((unsigned long)vaddr, get_order(size));
861 }
862}