2 ** IA64 System Bus Adapter (SBA) I/O MMU manager
4 ** (c) Copyright 2002-2005 Alex Williamson
5 ** (c) Copyright 2002-2003 Grant Grundler
6 ** (c) Copyright 2002-2005 Hewlett-Packard Company
8 ** Portions (c) 2000 Grant Grundler (from parisc I/O MMU code)
9 ** Portions (c) 1999 Dave S. Miller (from sparc64 I/O MMU code)
11 ** This program is free software; you can redistribute it and/or modify
12 ** it under the terms of the GNU General Public License as published by
13 ** the Free Software Foundation; either version 2 of the License, or
14 ** (at your option) any later version.
17 ** This module initializes the IOC (I/O Controller) found on HP
18 ** McKinley machines and their successors.
22 #include <linux/types.h>
23 #include <linux/kernel.h>
24 #include <linux/module.h>
25 #include <linux/spinlock.h>
26 #include <linux/slab.h>
27 #include <linux/init.h>
29 #include <linux/string.h>
30 #include <linux/pci.h>
31 #include <linux/proc_fs.h>
32 #include <linux/seq_file.h>
33 #include <linux/acpi.h>
34 #include <linux/efi.h>
35 #include <linux/nodemask.h>
36 #include <linux/bitops.h> /* hweight64() */
37 #include <linux/crash_dump.h>
38 #include <linux/iommu-helper.h>
40 #include <asm/delay.h> /* ia64_get_itc() */
42 #include <asm/page.h> /* PAGE_OFFSET */
44 #include <asm/system.h> /* wmb() */
46 #include <asm/acpi-ext.h>
48 extern int swiotlb_late_init_with_default_size (size_t size
);
53 ** Enabling timing search of the pdir resource map. Output in /proc.
54 ** Disabled by default to optimize performance.
56 #undef PDIR_SEARCH_TIMING
59 ** This option allows cards capable of 64bit DMA to bypass the IOMMU. If
60 ** not defined, all DMA will be 32bit and go through the TLB.
61 ** There's potentially a conflict in the bio merge code with us
62 ** advertising an iommu, but then bypassing it. Since I/O MMU bypassing
63 ** appears to give more performance than bio-level virtual merging, we'll
64 ** do the former for now. NOTE: BYPASS_SG also needs to be undef'd to
65 ** completely restrict DMA to the IOMMU.
67 #define ALLOW_IOV_BYPASS
70 ** This option specifically allows/disallows bypassing scatterlists with
71 ** multiple entries. Coalescing these entries can allow better DMA streaming
72 ** and in some cases shows better performance than entirely bypassing the
73 ** IOMMU. Performance increase on the order of 1-2% sequential output/input
74 ** using bonnie++ on a RAID0 MD device (sym2 & mpt).
76 #undef ALLOW_IOV_BYPASS_SG
79 ** If a device prefetches beyond the end of a valid pdir entry, it will cause
80 ** a hard failure, ie. MCA. Version 3.0 and later of the zx1 LBA should
81 ** disconnect on 4k boundaries and prevent such issues. If the device is
82 ** particularly aggressive, this option will keep the entire pdir valid such
83 ** that prefetching will hit a valid address. This could severely impact
84 ** error containment, and is therefore off by default. The page that is
85 ** used for spill-over is poisoned, so that should help debugging somewhat.
87 #undef FULL_VALID_PDIR
89 #define ENABLE_MARK_CLEAN
92 ** The number of debug flags is a clue - this code is fragile. NOTE: since
93 ** tightening the use of res_lock the resource bitmap and actual pdir are no
94 ** longer guaranteed to stay in sync. The sanity checking code isn't going to
99 #undef DEBUG_SBA_RUN_SG
100 #undef DEBUG_SBA_RESOURCE
101 #undef ASSERT_PDIR_SANITY
102 #undef DEBUG_LARGE_SG_ENTRIES
105 #if defined(FULL_VALID_PDIR) && defined(ASSERT_PDIR_SANITY)
106 #error FULL_VALID_PDIR and ASSERT_PDIR_SANITY are mutually exclusive
109 #define SBA_INLINE __inline__
110 /* #define SBA_INLINE */
112 #ifdef DEBUG_SBA_INIT
113 #define DBG_INIT(x...) printk(x)
115 #define DBG_INIT(x...)
119 #define DBG_RUN(x...) printk(x)
121 #define DBG_RUN(x...)
124 #ifdef DEBUG_SBA_RUN_SG
125 #define DBG_RUN_SG(x...) printk(x)
127 #define DBG_RUN_SG(x...)
131 #ifdef DEBUG_SBA_RESOURCE
132 #define DBG_RES(x...) printk(x)
134 #define DBG_RES(x...)
138 #define DBG_BYPASS(x...) printk(x)
140 #define DBG_BYPASS(x...)
143 #ifdef ASSERT_PDIR_SANITY
144 #define ASSERT(expr) \
146 printk( "\n" __FILE__ ":%d: Assertion " #expr " failed!\n",__LINE__); \
154 ** The number of pdir entries to "free" before issuing
155 ** a read to PCOM register to flush out PCOM writes.
156 ** Interacts with allocation granularity (ie 4 or 8 entries
157 ** allocated and free'd/purged at a time might make this
158 ** less interesting).
160 #define DELAYED_RESOURCE_CNT 64
162 #define PCI_DEVICE_ID_HP_SX2000_IOC 0x12ec
164 #define ZX1_IOC_ID ((PCI_DEVICE_ID_HP_ZX1_IOC << 16) | PCI_VENDOR_ID_HP)
165 #define ZX2_IOC_ID ((PCI_DEVICE_ID_HP_ZX2_IOC << 16) | PCI_VENDOR_ID_HP)
166 #define REO_IOC_ID ((PCI_DEVICE_ID_HP_REO_IOC << 16) | PCI_VENDOR_ID_HP)
167 #define SX1000_IOC_ID ((PCI_DEVICE_ID_HP_SX1000_IOC << 16) | PCI_VENDOR_ID_HP)
168 #define SX2000_IOC_ID ((PCI_DEVICE_ID_HP_SX2000_IOC << 16) | PCI_VENDOR_ID_HP)
170 #define ZX1_IOC_OFFSET 0x1000 /* ACPI reports SBA, we want IOC */
172 #define IOC_FUNC_ID 0x000
173 #define IOC_FCLASS 0x008 /* function class, bist, header, rev... */
174 #define IOC_IBASE 0x300 /* IO TLB */
175 #define IOC_IMASK 0x308
176 #define IOC_PCOM 0x310
177 #define IOC_TCNFG 0x318
178 #define IOC_PDIR_BASE 0x320
180 #define IOC_ROPE0_CFG 0x500
181 #define IOC_ROPE_AO 0x10 /* Allow "Relaxed Ordering" */
184 /* AGP GART driver looks for this */
185 #define ZX1_SBA_IOMMU_COOKIE 0x0000badbadc0ffeeUL
188 ** The zx1 IOC supports 4/8/16/64KB page sizes (see TCNFG register)
190 ** Some IOCs (sx1000) can run at the above pages sizes, but are
191 ** really only supported using the IOC at a 4k page size.
193 ** iovp_size could only be greater than PAGE_SIZE if we are
194 ** confident the drivers really only touch the next physical
195 ** page iff that driver instance owns it.
197 static unsigned long iovp_size
;
198 static unsigned long iovp_shift
;
199 static unsigned long iovp_mask
;
202 void __iomem
*ioc_hpa
; /* I/O MMU base address */
203 char *res_map
; /* resource map, bit == pdir entry */
204 u64
*pdir_base
; /* physical base address */
205 unsigned long ibase
; /* pdir IOV Space base */
206 unsigned long imask
; /* pdir IOV Space mask */
208 unsigned long *res_hint
; /* next avail IOVP - circular search */
209 unsigned long dma_mask
;
210 spinlock_t res_lock
; /* protects the resource bitmap, but must be held when */
211 /* clearing pdir to prevent races with allocations. */
212 unsigned int res_bitshift
; /* from the RIGHT! */
213 unsigned int res_size
; /* size of resource map in bytes */
215 unsigned int node
; /* node where this IOC lives */
217 #if DELAYED_RESOURCE_CNT > 0
218 spinlock_t saved_lock
; /* may want to try to get this on a separate cacheline */
219 /* than res_lock for bigger systems. */
221 struct sba_dma_pair
{
224 } saved
[DELAYED_RESOURCE_CNT
];
227 #ifdef PDIR_SEARCH_TIMING
228 #define SBA_SEARCH_SAMPLE 0x100
229 unsigned long avg_search
[SBA_SEARCH_SAMPLE
];
230 unsigned long avg_idx
; /* current index into avg_search */
233 /* Stuff we don't need in performance path */
234 struct ioc
*next
; /* list of IOC's in system */
235 acpi_handle handle
; /* for multiple IOC's */
237 unsigned int func_id
;
238 unsigned int rev
; /* HW revision of chip */
240 unsigned int pdir_size
; /* in bytes, determined by IOV Space size */
241 struct pci_dev
*sac_only_dev
;
244 static struct ioc
*ioc_list
;
245 static int reserve_sba_gart
= 1;
247 static SBA_INLINE
void sba_mark_invalid(struct ioc
*, dma_addr_t
, size_t);
248 static SBA_INLINE
void sba_free_range(struct ioc
*, dma_addr_t
, size_t);
250 #define sba_sg_address(sg) sg_virt((sg))
252 #ifdef FULL_VALID_PDIR
253 static u64 prefetch_spill_page
;
257 # define GET_IOC(dev) (((dev)->bus == &pci_bus_type) \
258 ? ((struct ioc *) PCI_CONTROLLER(to_pci_dev(dev))->iommu) : NULL)
260 # define GET_IOC(dev) NULL
264 ** DMA_CHUNK_SIZE is used by the SCSI mid-layer to break up
265 ** (or rather not merge) DMAs into manageable chunks.
266 ** On parisc, this is more of the software/tuning constraint
267 ** rather than the HW. I/O MMU allocation algorithms can be
268 ** faster with smaller sizes (to some degree).
270 #define DMA_CHUNK_SIZE (BITS_PER_LONG*iovp_size)
272 #define ROUNDUP(x,y) ((x + ((y)-1)) & ~((y)-1))
274 /************************************
275 ** SBA register read and write support
277 ** BE WARNED: register writes are posted.
278 ** (ie follow writes which must reach HW with a read)
281 #define READ_REG(addr) __raw_readq(addr)
282 #define WRITE_REG(val, addr) __raw_writeq(val, addr)
284 #ifdef DEBUG_SBA_INIT
287 * sba_dump_tlb - debugging only - print IOMMU operating parameters
288 * @hpa: base address of the IOMMU
290 * Print the size/location of the IO MMU PDIR.
293 sba_dump_tlb(char *hpa
)
295 DBG_INIT("IO TLB at 0x%p\n", (void *)hpa
);
296 DBG_INIT("IOC_IBASE : %016lx\n", READ_REG(hpa
+IOC_IBASE
));
297 DBG_INIT("IOC_IMASK : %016lx\n", READ_REG(hpa
+IOC_IMASK
));
298 DBG_INIT("IOC_TCNFG : %016lx\n", READ_REG(hpa
+IOC_TCNFG
));
299 DBG_INIT("IOC_PDIR_BASE: %016lx\n", READ_REG(hpa
+IOC_PDIR_BASE
));
305 #ifdef ASSERT_PDIR_SANITY
308 * sba_dump_pdir_entry - debugging only - print one IOMMU PDIR entry
309 * @ioc: IO MMU structure which owns the pdir we are interested in.
310 * @msg: text to print ont the output line.
313 * Print one entry of the IO MMU PDIR in human readable form.
316 sba_dump_pdir_entry(struct ioc
*ioc
, char *msg
, uint pide
)
318 /* start printing from lowest pde in rval */
319 u64
*ptr
= &ioc
->pdir_base
[pide
& ~(BITS_PER_LONG
- 1)];
320 unsigned long *rptr
= (unsigned long *) &ioc
->res_map
[(pide
>>3) & -sizeof(unsigned long)];
323 printk(KERN_DEBUG
"SBA: %s rp %p bit %d rval 0x%lx\n",
324 msg
, rptr
, pide
& (BITS_PER_LONG
- 1), *rptr
);
327 while (rcnt
< BITS_PER_LONG
) {
328 printk(KERN_DEBUG
"%s %2d %p %016Lx\n",
329 (rcnt
== (pide
& (BITS_PER_LONG
- 1)))
331 rcnt
, ptr
, (unsigned long long) *ptr
);
335 printk(KERN_DEBUG
"%s", msg
);
340 * sba_check_pdir - debugging only - consistency checker
341 * @ioc: IO MMU structure which owns the pdir we are interested in.
342 * @msg: text to print ont the output line.
344 * Verify the resource map and pdir state is consistent
347 sba_check_pdir(struct ioc
*ioc
, char *msg
)
349 u64
*rptr_end
= (u64
*) &(ioc
->res_map
[ioc
->res_size
]);
350 u64
*rptr
= (u64
*) ioc
->res_map
; /* resource map ptr */
351 u64
*pptr
= ioc
->pdir_base
; /* pdir ptr */
354 while (rptr
< rptr_end
) {
356 int rcnt
; /* number of bits we might check */
362 /* Get last byte and highest bit from that */
363 u32 pde
= ((u32
)((*pptr
>> (63)) & 0x1));
364 if ((rval
& 0x1) ^ pde
)
367 ** BUMMER! -- res_map != pdir --
368 ** Dump rval and matching pdir entries
370 sba_dump_pdir_entry(ioc
, msg
, pide
);
374 rval
>>= 1; /* try the next bit */
378 rptr
++; /* look at next word of res_map */
380 /* It'd be nice if we always got here :^) */
386 * sba_dump_sg - debugging only - print Scatter-Gather list
387 * @ioc: IO MMU structure which owns the pdir we are interested in.
388 * @startsg: head of the SG list
389 * @nents: number of entries in SG list
391 * print the SG list so we can verify it's correct by hand.
394 sba_dump_sg( struct ioc
*ioc
, struct scatterlist
*startsg
, int nents
)
396 while (nents
-- > 0) {
397 printk(KERN_DEBUG
" %d : DMA %08lx/%05x CPU %p\n", nents
,
398 startsg
->dma_address
, startsg
->dma_length
,
399 sba_sg_address(startsg
));
400 startsg
= sg_next(startsg
);
405 sba_check_sg( struct ioc
*ioc
, struct scatterlist
*startsg
, int nents
)
407 struct scatterlist
*the_sg
= startsg
;
408 int the_nents
= nents
;
410 while (the_nents
-- > 0) {
411 if (sba_sg_address(the_sg
) == 0x0UL
)
412 sba_dump_sg(NULL
, startsg
, nents
);
413 the_sg
= sg_next(the_sg
);
417 #endif /* ASSERT_PDIR_SANITY */
422 /**************************************************************
424 * I/O Pdir Resource Management
426 * Bits set in the resource map are in use.
427 * Each bit can represent a number of pages.
428 * LSbs represent lower addresses (IOVA's).
430 ***************************************************************/
431 #define PAGES_PER_RANGE 1 /* could increase this to 4 or 8 if needed */
433 /* Convert from IOVP to IOVA and vice versa. */
434 #define SBA_IOVA(ioc,iovp,offset) ((ioc->ibase) | (iovp) | (offset))
435 #define SBA_IOVP(ioc,iova) ((iova) & ~(ioc->ibase))
437 #define PDIR_ENTRY_SIZE sizeof(u64)
439 #define PDIR_INDEX(iovp) ((iovp)>>iovp_shift)
441 #define RESMAP_MASK(n) ~(~0UL << (n))
442 #define RESMAP_IDX_MASK (sizeof(unsigned long) - 1)
446 * For most cases the normal get_order is sufficient, however it limits us
447 * to PAGE_SIZE being the minimum mapping alignment and TC flush granularity.
448 * It only incurs about 1 clock cycle to use this one with the static variable
449 * and makes the code more intuitive.
451 static SBA_INLINE
int
452 get_iovp_order (unsigned long size
)
454 long double d
= size
- 1;
457 order
= ia64_getf_exp(d
);
458 order
= order
- iovp_shift
- 0xffff + 1;
464 static unsigned long ptr_to_pide(struct ioc
*ioc
, unsigned long *res_ptr
,
465 unsigned int bitshiftcnt
)
467 return (((unsigned long)res_ptr
- (unsigned long)ioc
->res_map
) << 3)
472 * sba_search_bitmap - find free space in IO PDIR resource bitmap
473 * @ioc: IO MMU structure which owns the pdir we are interested in.
474 * @bits_wanted: number of entries we need.
475 * @use_hint: use res_hint to indicate where to start looking
477 * Find consecutive free bits in resource bitmap.
478 * Each bit represents one entry in the IO Pdir.
479 * Cool perf optimization: search for log2(size) bits at a time.
481 static SBA_INLINE
unsigned long
482 sba_search_bitmap(struct ioc
*ioc
, struct device
*dev
,
483 unsigned long bits_wanted
, int use_hint
)
485 unsigned long *res_ptr
;
486 unsigned long *res_end
= (unsigned long *) &(ioc
->res_map
[ioc
->res_size
]);
487 unsigned long flags
, pide
= ~0UL, tpide
;
488 unsigned long boundary_size
;
492 ASSERT(((unsigned long) ioc
->res_hint
& (sizeof(unsigned long) - 1UL)) == 0);
493 ASSERT(res_ptr
< res_end
);
495 boundary_size
= (unsigned long long)dma_get_seg_boundary(dev
) + 1;
496 boundary_size
= ALIGN(boundary_size
, 1ULL << iovp_shift
) >> iovp_shift
;
498 BUG_ON(ioc
->ibase
& ~iovp_mask
);
499 shift
= ioc
->ibase
>> iovp_shift
;
501 spin_lock_irqsave(&ioc
->res_lock
, flags
);
503 /* Allow caller to force a search through the entire resource space */
504 if (likely(use_hint
)) {
505 res_ptr
= ioc
->res_hint
;
507 res_ptr
= (ulong
*)ioc
->res_map
;
508 ioc
->res_bitshift
= 0;
512 * N.B. REO/Grande defect AR2305 can cause TLB fetch timeouts
513 * if a TLB entry is purged while in use. sba_mark_invalid()
514 * purges IOTLB entries in power-of-two sizes, so we also
515 * allocate IOVA space in power-of-two sizes.
517 bits_wanted
= 1UL << get_iovp_order(bits_wanted
<< iovp_shift
);
519 if (likely(bits_wanted
== 1)) {
520 unsigned int bitshiftcnt
;
521 for(; res_ptr
< res_end
; res_ptr
++) {
522 if (likely(*res_ptr
!= ~0UL)) {
523 bitshiftcnt
= ffz(*res_ptr
);
524 *res_ptr
|= (1UL << bitshiftcnt
);
525 pide
= ptr_to_pide(ioc
, res_ptr
, bitshiftcnt
);
526 ioc
->res_bitshift
= bitshiftcnt
+ bits_wanted
;
534 if (likely(bits_wanted
<= BITS_PER_LONG
/2)) {
536 ** Search the resource bit map on well-aligned values.
537 ** "o" is the alignment.
538 ** We need the alignment to invalidate I/O TLB using
539 ** SBA HW features in the unmap path.
541 unsigned long o
= 1 << get_iovp_order(bits_wanted
<< iovp_shift
);
542 uint bitshiftcnt
= ROUNDUP(ioc
->res_bitshift
, o
);
543 unsigned long mask
, base_mask
;
545 base_mask
= RESMAP_MASK(bits_wanted
);
546 mask
= base_mask
<< bitshiftcnt
;
548 DBG_RES("%s() o %ld %p", __func__
, o
, res_ptr
);
549 for(; res_ptr
< res_end
; res_ptr
++)
551 DBG_RES(" %p %lx %lx\n", res_ptr
, mask
, *res_ptr
);
553 for (; mask
; mask
<<= o
, bitshiftcnt
+= o
) {
554 tpide
= ptr_to_pide(ioc
, res_ptr
, bitshiftcnt
);
555 ret
= iommu_is_span_boundary(tpide
, bits_wanted
,
558 if ((0 == ((*res_ptr
) & mask
)) && !ret
) {
559 *res_ptr
|= mask
; /* mark resources busy! */
561 ioc
->res_bitshift
= bitshiftcnt
+ bits_wanted
;
575 qwords
= bits_wanted
>> 6; /* /64 */
576 bits
= bits_wanted
- (qwords
* BITS_PER_LONG
);
578 end
= res_end
- qwords
;
580 for (; res_ptr
< end
; res_ptr
++) {
581 tpide
= ptr_to_pide(ioc
, res_ptr
, 0);
582 ret
= iommu_is_span_boundary(tpide
, bits_wanted
,
583 shift
, boundary_size
);
586 for (i
= 0 ; i
< qwords
; i
++) {
590 if (bits
&& res_ptr
[i
] && (__ffs(res_ptr
[i
]) < bits
))
593 /* Found it, mark it */
594 for (i
= 0 ; i
< qwords
; i
++)
596 res_ptr
[i
] |= RESMAP_MASK(bits
);
600 ioc
->res_bitshift
= bits
;
608 prefetch(ioc
->res_map
);
609 ioc
->res_hint
= (unsigned long *) ioc
->res_map
;
610 ioc
->res_bitshift
= 0;
611 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
615 ioc
->res_hint
= res_ptr
;
616 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
622 * sba_alloc_range - find free bits and mark them in IO PDIR resource bitmap
623 * @ioc: IO MMU structure which owns the pdir we are interested in.
624 * @size: number of bytes to create a mapping for
626 * Given a size, find consecutive unmarked and then mark those bits in the
630 sba_alloc_range(struct ioc
*ioc
, struct device
*dev
, size_t size
)
632 unsigned int pages_needed
= size
>> iovp_shift
;
633 #ifdef PDIR_SEARCH_TIMING
634 unsigned long itc_start
;
638 ASSERT(pages_needed
);
639 ASSERT(0 == (size
& ~iovp_mask
));
641 #ifdef PDIR_SEARCH_TIMING
642 itc_start
= ia64_get_itc();
645 ** "seek and ye shall find"...praying never hurts either...
647 pide
= sba_search_bitmap(ioc
, dev
, pages_needed
, 1);
648 if (unlikely(pide
>= (ioc
->res_size
<< 3))) {
649 pide
= sba_search_bitmap(ioc
, dev
, pages_needed
, 0);
650 if (unlikely(pide
>= (ioc
->res_size
<< 3))) {
651 #if DELAYED_RESOURCE_CNT > 0
655 ** With delayed resource freeing, we can give this one more shot. We're
656 ** getting close to being in trouble here, so do what we can to make this
659 spin_lock_irqsave(&ioc
->saved_lock
, flags
);
660 if (ioc
->saved_cnt
> 0) {
661 struct sba_dma_pair
*d
;
662 int cnt
= ioc
->saved_cnt
;
664 d
= &(ioc
->saved
[ioc
->saved_cnt
- 1]);
666 spin_lock(&ioc
->res_lock
);
668 sba_mark_invalid(ioc
, d
->iova
, d
->size
);
669 sba_free_range(ioc
, d
->iova
, d
->size
);
673 READ_REG(ioc
->ioc_hpa
+IOC_PCOM
); /* flush purges */
674 spin_unlock(&ioc
->res_lock
);
676 spin_unlock_irqrestore(&ioc
->saved_lock
, flags
);
678 pide
= sba_search_bitmap(ioc
, dev
, pages_needed
, 0);
679 if (unlikely(pide
>= (ioc
->res_size
<< 3)))
680 panic(__FILE__
": I/O MMU @ %p is out of mapping resources\n",
683 panic(__FILE__
": I/O MMU @ %p is out of mapping resources\n",
689 #ifdef PDIR_SEARCH_TIMING
690 ioc
->avg_search
[ioc
->avg_idx
++] = (ia64_get_itc() - itc_start
) / pages_needed
;
691 ioc
->avg_idx
&= SBA_SEARCH_SAMPLE
- 1;
694 prefetchw(&(ioc
->pdir_base
[pide
]));
696 #ifdef ASSERT_PDIR_SANITY
697 /* verify the first enable bit is clear */
698 if(0x00 != ((u8
*) ioc
->pdir_base
)[pide
*PDIR_ENTRY_SIZE
+ 7]) {
699 sba_dump_pdir_entry(ioc
, "sba_search_bitmap() botched it?", pide
);
703 DBG_RES("%s(%x) %d -> %lx hint %x/%x\n",
704 __func__
, size
, pages_needed
, pide
,
705 (uint
) ((unsigned long) ioc
->res_hint
- (unsigned long) ioc
->res_map
),
713 * sba_free_range - unmark bits in IO PDIR resource bitmap
714 * @ioc: IO MMU structure which owns the pdir we are interested in.
715 * @iova: IO virtual address which was previously allocated.
716 * @size: number of bytes to create a mapping for
718 * clear bits in the ioc's resource map
720 static SBA_INLINE
void
721 sba_free_range(struct ioc
*ioc
, dma_addr_t iova
, size_t size
)
723 unsigned long iovp
= SBA_IOVP(ioc
, iova
);
724 unsigned int pide
= PDIR_INDEX(iovp
);
725 unsigned int ridx
= pide
>> 3; /* convert bit to byte address */
726 unsigned long *res_ptr
= (unsigned long *) &((ioc
)->res_map
[ridx
& ~RESMAP_IDX_MASK
]);
727 int bits_not_wanted
= size
>> iovp_shift
;
730 /* Round up to power-of-two size: see AR2305 note above */
731 bits_not_wanted
= 1UL << get_iovp_order(bits_not_wanted
<< iovp_shift
);
732 for (; bits_not_wanted
> 0 ; res_ptr
++) {
734 if (unlikely(bits_not_wanted
> BITS_PER_LONG
)) {
736 /* these mappings start 64bit aligned */
738 bits_not_wanted
-= BITS_PER_LONG
;
739 pide
+= BITS_PER_LONG
;
743 /* 3-bits "bit" address plus 2 (or 3) bits for "byte" == bit in word */
744 m
= RESMAP_MASK(bits_not_wanted
) << (pide
& (BITS_PER_LONG
- 1));
747 DBG_RES("%s( ,%x,%x) %x/%lx %x %p %lx\n", __func__
, (uint
) iova
, size
,
748 bits_not_wanted
, m
, pide
, res_ptr
, *res_ptr
);
751 ASSERT(bits_not_wanted
);
752 ASSERT((*res_ptr
& m
) == m
); /* verify same bits are set */
759 /**************************************************************
761 * "Dynamic DMA Mapping" support (aka "Coherent I/O")
763 ***************************************************************/
766 * sba_io_pdir_entry - fill in one IO PDIR entry
767 * @pdir_ptr: pointer to IO PDIR entry
768 * @vba: Virtual CPU address of buffer to map
770 * SBA Mapping Routine
772 * Given a virtual address (vba, arg1) sba_io_pdir_entry()
773 * loads the I/O PDIR entry pointed to by pdir_ptr (arg0).
774 * Each IO Pdir entry consists of 8 bytes as shown below
778 * +-+---------------------+----------------------------------+----+--------+
779 * |V| U | PPN[39:12] | U | FF |
780 * +-+---------------------+----------------------------------+----+--------+
784 * PPN == Physical Page Number
786 * The physical address fields are filled with the results of virt_to_phys()
791 #define sba_io_pdir_entry(pdir_ptr, vba) *pdir_ptr = ((vba & ~0xE000000000000FFFULL) \
792 | 0x8000000000000000ULL)
795 sba_io_pdir_entry(u64
*pdir_ptr
, unsigned long vba
)
797 *pdir_ptr
= ((vba
& ~0xE000000000000FFFULL
) | 0x80000000000000FFULL
);
801 #ifdef ENABLE_MARK_CLEAN
803 * Since DMA is i-cache coherent, any (complete) pages that were written via
804 * DMA can be marked as "clean" so that lazy_mmu_prot_update() doesn't have to
805 * flush them when they get mapped into an executable vm-area.
808 mark_clean (void *addr
, size_t size
)
810 unsigned long pg_addr
, end
;
812 pg_addr
= PAGE_ALIGN((unsigned long) addr
);
813 end
= (unsigned long) addr
+ size
;
814 while (pg_addr
+ PAGE_SIZE
<= end
) {
815 struct page
*page
= virt_to_page((void *)pg_addr
);
816 set_bit(PG_arch_1
, &page
->flags
);
817 pg_addr
+= PAGE_SIZE
;
823 * sba_mark_invalid - invalidate one or more IO PDIR entries
824 * @ioc: IO MMU structure which owns the pdir we are interested in.
825 * @iova: IO Virtual Address mapped earlier
826 * @byte_cnt: number of bytes this mapping covers.
828 * Marking the IO PDIR entry(ies) as Invalid and invalidate
829 * corresponding IO TLB entry. The PCOM (Purge Command Register)
830 * is to purge stale entries in the IO TLB when unmapping entries.
832 * The PCOM register supports purging of multiple pages, with a minium
833 * of 1 page and a maximum of 2GB. Hardware requires the address be
834 * aligned to the size of the range being purged. The size of the range
835 * must be a power of 2. The "Cool perf optimization" in the
836 * allocation routine helps keep that true.
838 static SBA_INLINE
void
839 sba_mark_invalid(struct ioc
*ioc
, dma_addr_t iova
, size_t byte_cnt
)
841 u32 iovp
= (u32
) SBA_IOVP(ioc
,iova
);
843 int off
= PDIR_INDEX(iovp
);
845 /* Must be non-zero and rounded up */
846 ASSERT(byte_cnt
> 0);
847 ASSERT(0 == (byte_cnt
& ~iovp_mask
));
849 #ifdef ASSERT_PDIR_SANITY
850 /* Assert first pdir entry is set */
851 if (!(ioc
->pdir_base
[off
] >> 60)) {
852 sba_dump_pdir_entry(ioc
,"sba_mark_invalid()", PDIR_INDEX(iovp
));
856 if (byte_cnt
<= iovp_size
)
858 ASSERT(off
< ioc
->pdir_size
);
860 iovp
|= iovp_shift
; /* set "size" field for PCOM */
862 #ifndef FULL_VALID_PDIR
864 ** clear I/O PDIR entry "valid" bit
865 ** Do NOT clear the rest - save it for debugging.
866 ** We should only clear bits that have previously
869 ioc
->pdir_base
[off
] &= ~(0x80000000000000FFULL
);
872 ** If we want to maintain the PDIR as valid, put in
873 ** the spill page so devices prefetching won't
874 ** cause a hard fail.
876 ioc
->pdir_base
[off
] = (0x80000000000000FFULL
| prefetch_spill_page
);
879 u32 t
= get_iovp_order(byte_cnt
) + iovp_shift
;
882 ASSERT(t
<= 31); /* 2GB! Max value of "size" field */
885 /* verify this pdir entry is enabled */
886 ASSERT(ioc
->pdir_base
[off
] >> 63);
887 #ifndef FULL_VALID_PDIR
888 /* clear I/O Pdir entry "valid" bit first */
889 ioc
->pdir_base
[off
] &= ~(0x80000000000000FFULL
);
891 ioc
->pdir_base
[off
] = (0x80000000000000FFULL
| prefetch_spill_page
);
894 byte_cnt
-= iovp_size
;
895 } while (byte_cnt
> 0);
898 WRITE_REG(iovp
| ioc
->ibase
, ioc
->ioc_hpa
+IOC_PCOM
);
902 * sba_map_single_attrs - map one buffer and return IOVA for DMA
903 * @dev: instance of PCI owned by the driver that's asking.
904 * @addr: driver buffer to map.
905 * @size: number of bytes to map in driver buffer.
907 * @attrs: optional dma attributes
909 * See Documentation/DMA-mapping.txt
912 sba_map_single_attrs(struct device
*dev
, void *addr
, size_t size
, int dir
,
913 struct dma_attrs
*attrs
)
920 #ifdef ASSERT_PDIR_SANITY
923 #ifdef ALLOW_IOV_BYPASS
924 unsigned long pci_addr
= virt_to_phys(addr
);
927 #ifdef ALLOW_IOV_BYPASS
928 ASSERT(to_pci_dev(dev
)->dma_mask
);
930 ** Check if the PCI device can DMA to ptr... if so, just return ptr
932 if (likely((pci_addr
& ~to_pci_dev(dev
)->dma_mask
) == 0)) {
934 ** Device is bit capable of DMA'ing to the buffer...
935 ** just return the PCI address of ptr
937 DBG_BYPASS("sba_map_single_attrs() bypass mask/addr: "
939 to_pci_dev(dev
)->dma_mask
, pci_addr
);
946 prefetch(ioc
->res_hint
);
949 ASSERT(size
<= DMA_CHUNK_SIZE
);
951 /* save offset bits */
952 offset
= ((dma_addr_t
) (long) addr
) & ~iovp_mask
;
954 /* round up to nearest iovp_size */
955 size
= (size
+ offset
+ ~iovp_mask
) & iovp_mask
;
957 #ifdef ASSERT_PDIR_SANITY
958 spin_lock_irqsave(&ioc
->res_lock
, flags
);
959 if (sba_check_pdir(ioc
,"Check before sba_map_single_attrs()"))
960 panic("Sanity check failed");
961 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
964 pide
= sba_alloc_range(ioc
, dev
, size
);
966 iovp
= (dma_addr_t
) pide
<< iovp_shift
;
968 DBG_RUN("%s() 0x%p -> 0x%lx\n", __func__
, addr
, (long) iovp
| offset
);
970 pdir_start
= &(ioc
->pdir_base
[pide
]);
973 ASSERT(((u8
*)pdir_start
)[7] == 0); /* verify availability */
974 sba_io_pdir_entry(pdir_start
, (unsigned long) addr
);
976 DBG_RUN(" pdir 0x%p %lx\n", pdir_start
, *pdir_start
);
982 /* force pdir update */
985 /* form complete address */
986 #ifdef ASSERT_PDIR_SANITY
987 spin_lock_irqsave(&ioc
->res_lock
, flags
);
988 sba_check_pdir(ioc
,"Check after sba_map_single_attrs()");
989 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
991 return SBA_IOVA(ioc
, iovp
, offset
);
993 EXPORT_SYMBOL(sba_map_single_attrs
);
995 #ifdef ENABLE_MARK_CLEAN
996 static SBA_INLINE
void
997 sba_mark_clean(struct ioc
*ioc
, dma_addr_t iova
, size_t size
)
999 u32 iovp
= (u32
) SBA_IOVP(ioc
,iova
);
1000 int off
= PDIR_INDEX(iovp
);
1003 if (size
<= iovp_size
) {
1004 addr
= phys_to_virt(ioc
->pdir_base
[off
] &
1005 ~0xE000000000000FFFULL
);
1006 mark_clean(addr
, size
);
1009 addr
= phys_to_virt(ioc
->pdir_base
[off
] &
1010 ~0xE000000000000FFFULL
);
1011 mark_clean(addr
, min(size
, iovp_size
));
1020 * sba_unmap_single_attrs - unmap one IOVA and free resources
1021 * @dev: instance of PCI owned by the driver that's asking.
1022 * @iova: IOVA of driver buffer previously mapped.
1023 * @size: number of bytes mapped in driver buffer.
1024 * @dir: R/W or both.
1025 * @attrs: optional dma attributes
1027 * See Documentation/DMA-mapping.txt
1029 void sba_unmap_single_attrs(struct device
*dev
, dma_addr_t iova
, size_t size
,
1030 int dir
, struct dma_attrs
*attrs
)
1033 #if DELAYED_RESOURCE_CNT > 0
1034 struct sba_dma_pair
*d
;
1036 unsigned long flags
;
1042 #ifdef ALLOW_IOV_BYPASS
1043 if (likely((iova
& ioc
->imask
) != ioc
->ibase
)) {
1045 ** Address does not fall w/in IOVA, must be bypassing
1047 DBG_BYPASS("sba_unmap_single_atttrs() bypass addr: 0x%lx\n",
1050 #ifdef ENABLE_MARK_CLEAN
1051 if (dir
== DMA_FROM_DEVICE
) {
1052 mark_clean(phys_to_virt(iova
), size
);
1058 offset
= iova
& ~iovp_mask
;
1060 DBG_RUN("%s() iovp 0x%lx/%x\n", __func__
, (long) iova
, size
);
1062 iova
^= offset
; /* clear offset bits */
1064 size
= ROUNDUP(size
, iovp_size
);
1066 #ifdef ENABLE_MARK_CLEAN
1067 if (dir
== DMA_FROM_DEVICE
)
1068 sba_mark_clean(ioc
, iova
, size
);
1071 #if DELAYED_RESOURCE_CNT > 0
1072 spin_lock_irqsave(&ioc
->saved_lock
, flags
);
1073 d
= &(ioc
->saved
[ioc
->saved_cnt
]);
1076 if (unlikely(++(ioc
->saved_cnt
) >= DELAYED_RESOURCE_CNT
)) {
1077 int cnt
= ioc
->saved_cnt
;
1078 spin_lock(&ioc
->res_lock
);
1080 sba_mark_invalid(ioc
, d
->iova
, d
->size
);
1081 sba_free_range(ioc
, d
->iova
, d
->size
);
1085 READ_REG(ioc
->ioc_hpa
+IOC_PCOM
); /* flush purges */
1086 spin_unlock(&ioc
->res_lock
);
1088 spin_unlock_irqrestore(&ioc
->saved_lock
, flags
);
1089 #else /* DELAYED_RESOURCE_CNT == 0 */
1090 spin_lock_irqsave(&ioc
->res_lock
, flags
);
1091 sba_mark_invalid(ioc
, iova
, size
);
1092 sba_free_range(ioc
, iova
, size
);
1093 READ_REG(ioc
->ioc_hpa
+IOC_PCOM
); /* flush purges */
1094 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
1095 #endif /* DELAYED_RESOURCE_CNT == 0 */
1097 EXPORT_SYMBOL(sba_unmap_single_attrs
);
1100 * sba_alloc_coherent - allocate/map shared mem for DMA
1101 * @dev: instance of PCI owned by the driver that's asking.
1102 * @size: number of bytes mapped in driver buffer.
1103 * @dma_handle: IOVA of new buffer.
1105 * See Documentation/DMA-mapping.txt
1108 sba_alloc_coherent (struct device
*dev
, size_t size
, dma_addr_t
*dma_handle
, gfp_t flags
)
1119 page
= alloc_pages_node(ioc
->node
== MAX_NUMNODES
?
1120 numa_node_id() : ioc
->node
, flags
,
1123 if (unlikely(!page
))
1126 addr
= page_address(page
);
1129 addr
= (void *) __get_free_pages(flags
, get_order(size
));
1131 if (unlikely(!addr
))
1134 memset(addr
, 0, size
);
1135 *dma_handle
= virt_to_phys(addr
);
1137 #ifdef ALLOW_IOV_BYPASS
1138 ASSERT(dev
->coherent_dma_mask
);
1140 ** Check if the PCI device can DMA to ptr... if so, just return ptr
1142 if (likely((*dma_handle
& ~dev
->coherent_dma_mask
) == 0)) {
1143 DBG_BYPASS("sba_alloc_coherent() bypass mask/addr: 0x%lx/0x%lx\n",
1144 dev
->coherent_dma_mask
, *dma_handle
);
1151 * If device can't bypass or bypass is disabled, pass the 32bit fake
1152 * device to map single to get an iova mapping.
1154 *dma_handle
= sba_map_single_attrs(&ioc
->sac_only_dev
->dev
, addr
,
1162 * sba_free_coherent - free/unmap shared mem for DMA
1163 * @dev: instance of PCI owned by the driver that's asking.
1164 * @size: number of bytes mapped in driver buffer.
1165 * @vaddr: virtual address IOVA of "consistent" buffer.
1166 * @dma_handler: IO virtual address of "consistent" buffer.
1168 * See Documentation/DMA-mapping.txt
1170 void sba_free_coherent (struct device
*dev
, size_t size
, void *vaddr
, dma_addr_t dma_handle
)
1172 sba_unmap_single_attrs(dev
, dma_handle
, size
, 0, NULL
);
1173 free_pages((unsigned long) vaddr
, get_order(size
));
1178 ** Since 0 is a valid pdir_base index value, can't use that
1179 ** to determine if a value is valid or not. Use a flag to indicate
1180 ** the SG list entry contains a valid pdir index.
1182 #define PIDE_FLAG 0x1UL
1184 #ifdef DEBUG_LARGE_SG_ENTRIES
1185 int dump_run_sg
= 0;
1190 * sba_fill_pdir - write allocated SG entries into IO PDIR
1191 * @ioc: IO MMU structure which owns the pdir we are interested in.
1192 * @startsg: list of IOVA/size pairs
1193 * @nents: number of entries in startsg list
1195 * Take preprocessed SG list and write corresponding entries
1199 static SBA_INLINE
int
1202 struct scatterlist
*startsg
,
1205 struct scatterlist
*dma_sg
= startsg
; /* pointer to current DMA */
1208 unsigned long dma_offset
= 0;
1210 while (nents
-- > 0) {
1211 int cnt
= startsg
->dma_length
;
1212 startsg
->dma_length
= 0;
1214 #ifdef DEBUG_LARGE_SG_ENTRIES
1216 printk(" %2d : %08lx/%05x %p\n",
1217 nents
, startsg
->dma_address
, cnt
,
1218 sba_sg_address(startsg
));
1220 DBG_RUN_SG(" %d : %08lx/%05x %p\n",
1221 nents
, startsg
->dma_address
, cnt
,
1222 sba_sg_address(startsg
));
1225 ** Look for the start of a new DMA stream
1227 if (startsg
->dma_address
& PIDE_FLAG
) {
1228 u32 pide
= startsg
->dma_address
& ~PIDE_FLAG
;
1229 dma_offset
= (unsigned long) pide
& ~iovp_mask
;
1230 startsg
->dma_address
= 0;
1232 dma_sg
= sg_next(dma_sg
);
1233 dma_sg
->dma_address
= pide
| ioc
->ibase
;
1234 pdirp
= &(ioc
->pdir_base
[pide
>> iovp_shift
]);
1239 ** Look for a VCONTIG chunk
1242 unsigned long vaddr
= (unsigned long) sba_sg_address(startsg
);
1245 /* Since multiple Vcontig blocks could make up
1246 ** one DMA stream, *add* cnt to dma_len.
1248 dma_sg
->dma_length
+= cnt
;
1250 dma_offset
=0; /* only want offset on first chunk */
1251 cnt
= ROUNDUP(cnt
, iovp_size
);
1253 sba_io_pdir_entry(pdirp
, vaddr
);
1259 startsg
= sg_next(startsg
);
1261 /* force pdir update */
1264 #ifdef DEBUG_LARGE_SG_ENTRIES
1272 ** Two address ranges are DMA contiguous *iff* "end of prev" and
1273 ** "start of next" are both on an IOV page boundary.
1275 ** (shift left is a quick trick to mask off upper bits)
1277 #define DMA_CONTIG(__X, __Y) \
1278 (((((unsigned long) __X) | ((unsigned long) __Y)) << (BITS_PER_LONG - iovp_shift)) == 0UL)
1282 * sba_coalesce_chunks - preprocess the SG list
1283 * @ioc: IO MMU structure which owns the pdir we are interested in.
1284 * @startsg: list of IOVA/size pairs
1285 * @nents: number of entries in startsg list
1287 * First pass is to walk the SG list and determine where the breaks are
1288 * in the DMA stream. Allocates PDIR entries but does not fill them.
1289 * Returns the number of DMA chunks.
1291 * Doing the fill separate from the coalescing/allocation keeps the
1292 * code simpler. Future enhancement could make one pass through
1293 * the sglist do both.
1295 static SBA_INLINE
int
1296 sba_coalesce_chunks(struct ioc
*ioc
, struct device
*dev
,
1297 struct scatterlist
*startsg
,
1300 struct scatterlist
*vcontig_sg
; /* VCONTIG chunk head */
1301 unsigned long vcontig_len
; /* len of VCONTIG chunk */
1302 unsigned long vcontig_end
;
1303 struct scatterlist
*dma_sg
; /* next DMA stream head */
1304 unsigned long dma_offset
, dma_len
; /* start/len of DMA stream */
1306 unsigned int max_seg_size
= dma_get_max_seg_size(dev
);
1309 unsigned long vaddr
= (unsigned long) sba_sg_address(startsg
);
1312 ** Prepare for first/next DMA stream
1314 dma_sg
= vcontig_sg
= startsg
;
1315 dma_len
= vcontig_len
= vcontig_end
= startsg
->length
;
1316 vcontig_end
+= vaddr
;
1317 dma_offset
= vaddr
& ~iovp_mask
;
1319 /* PARANOID: clear entries */
1320 startsg
->dma_address
= startsg
->dma_length
= 0;
1323 ** This loop terminates one iteration "early" since
1324 ** it's always looking one "ahead".
1326 while (--nents
> 0) {
1327 unsigned long vaddr
; /* tmp */
1329 startsg
= sg_next(startsg
);
1332 startsg
->dma_address
= startsg
->dma_length
= 0;
1334 /* catch brokenness in SCSI layer */
1335 ASSERT(startsg
->length
<= DMA_CHUNK_SIZE
);
1338 ** First make sure current dma stream won't
1339 ** exceed DMA_CHUNK_SIZE if we coalesce the
1342 if (((dma_len
+ dma_offset
+ startsg
->length
+ ~iovp_mask
) & iovp_mask
)
1346 if (dma_len
+ startsg
->length
> max_seg_size
)
1350 ** Then look for virtually contiguous blocks.
1352 ** append the next transaction?
1354 vaddr
= (unsigned long) sba_sg_address(startsg
);
1355 if (vcontig_end
== vaddr
)
1357 vcontig_len
+= startsg
->length
;
1358 vcontig_end
+= startsg
->length
;
1359 dma_len
+= startsg
->length
;
1363 #ifdef DEBUG_LARGE_SG_ENTRIES
1364 dump_run_sg
= (vcontig_len
> iovp_size
);
1368 ** Not virtually contigous.
1369 ** Terminate prev chunk.
1370 ** Start a new chunk.
1372 ** Once we start a new VCONTIG chunk, dma_offset
1373 ** can't change. And we need the offset from the first
1374 ** chunk - not the last one. Ergo Successive chunks
1375 ** must start on page boundaries and dove tail
1376 ** with it's predecessor.
1378 vcontig_sg
->dma_length
= vcontig_len
;
1380 vcontig_sg
= startsg
;
1381 vcontig_len
= startsg
->length
;
1384 ** 3) do the entries end/start on page boundaries?
1385 ** Don't update vcontig_end until we've checked.
1387 if (DMA_CONTIG(vcontig_end
, vaddr
))
1389 vcontig_end
= vcontig_len
+ vaddr
;
1390 dma_len
+= vcontig_len
;
1398 ** End of DMA Stream
1399 ** Terminate last VCONTIG block.
1400 ** Allocate space for DMA stream.
1402 vcontig_sg
->dma_length
= vcontig_len
;
1403 dma_len
= (dma_len
+ dma_offset
+ ~iovp_mask
) & iovp_mask
;
1404 ASSERT(dma_len
<= DMA_CHUNK_SIZE
);
1405 dma_sg
->dma_address
= (dma_addr_t
) (PIDE_FLAG
1406 | (sba_alloc_range(ioc
, dev
, dma_len
) << iovp_shift
)
1416 * sba_map_sg - map Scatter/Gather list
1417 * @dev: instance of PCI owned by the driver that's asking.
1418 * @sglist: array of buffer/length pairs
1419 * @nents: number of entries in list
1420 * @dir: R/W or both.
1421 * @attrs: optional dma attributes
1423 * See Documentation/DMA-mapping.txt
1425 int sba_map_sg_attrs(struct device
*dev
, struct scatterlist
*sglist
, int nents
,
1426 int dir
, struct dma_attrs
*attrs
)
1429 int coalesced
, filled
= 0;
1430 #ifdef ASSERT_PDIR_SANITY
1431 unsigned long flags
;
1433 #ifdef ALLOW_IOV_BYPASS_SG
1434 struct scatterlist
*sg
;
1437 DBG_RUN_SG("%s() START %d entries\n", __func__
, nents
);
1441 #ifdef ALLOW_IOV_BYPASS_SG
1442 ASSERT(to_pci_dev(dev
)->dma_mask
);
1443 if (likely((ioc
->dma_mask
& ~to_pci_dev(dev
)->dma_mask
) == 0)) {
1444 for_each_sg(sglist
, sg
, nents
, filled
) {
1445 sg
->dma_length
= sg
->length
;
1446 sg
->dma_address
= virt_to_phys(sba_sg_address(sg
));
1451 /* Fast path single entry scatterlists. */
1453 sglist
->dma_length
= sglist
->length
;
1454 sglist
->dma_address
= sba_map_single_attrs(dev
, sba_sg_address(sglist
), sglist
->length
, dir
, attrs
);
1458 #ifdef ASSERT_PDIR_SANITY
1459 spin_lock_irqsave(&ioc
->res_lock
, flags
);
1460 if (sba_check_pdir(ioc
,"Check before sba_map_sg_attrs()"))
1462 sba_dump_sg(ioc
, sglist
, nents
);
1463 panic("Check before sba_map_sg_attrs()");
1465 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
1468 prefetch(ioc
->res_hint
);
1471 ** First coalesce the chunks and allocate I/O pdir space
1473 ** If this is one DMA stream, we can properly map using the
1474 ** correct virtual address associated with each DMA page.
1475 ** w/o this association, we wouldn't have coherent DMA!
1476 ** Access to the virtual address is what forces a two pass algorithm.
1478 coalesced
= sba_coalesce_chunks(ioc
, dev
, sglist
, nents
);
1481 ** Program the I/O Pdir
1483 ** map the virtual addresses to the I/O Pdir
1484 ** o dma_address will contain the pdir index
1485 ** o dma_len will contain the number of bytes to map
1486 ** o address contains the virtual address.
1488 filled
= sba_fill_pdir(ioc
, sglist
, nents
);
1490 #ifdef ASSERT_PDIR_SANITY
1491 spin_lock_irqsave(&ioc
->res_lock
, flags
);
1492 if (sba_check_pdir(ioc
,"Check after sba_map_sg_attrs()"))
1494 sba_dump_sg(ioc
, sglist
, nents
);
1495 panic("Check after sba_map_sg_attrs()\n");
1497 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
1500 ASSERT(coalesced
== filled
);
1501 DBG_RUN_SG("%s() DONE %d mappings\n", __func__
, filled
);
1505 EXPORT_SYMBOL(sba_map_sg_attrs
);
1508 * sba_unmap_sg_attrs - unmap Scatter/Gather list
1509 * @dev: instance of PCI owned by the driver that's asking.
1510 * @sglist: array of buffer/length pairs
1511 * @nents: number of entries in list
1512 * @dir: R/W or both.
1513 * @attrs: optional dma attributes
1515 * See Documentation/DMA-mapping.txt
1517 void sba_unmap_sg_attrs(struct device
*dev
, struct scatterlist
*sglist
,
1518 int nents
, int dir
, struct dma_attrs
*attrs
)
1520 #ifdef ASSERT_PDIR_SANITY
1522 unsigned long flags
;
1525 DBG_RUN_SG("%s() START %d entries, %p,%x\n",
1526 __func__
, nents
, sba_sg_address(sglist
), sglist
->length
);
1528 #ifdef ASSERT_PDIR_SANITY
1532 spin_lock_irqsave(&ioc
->res_lock
, flags
);
1533 sba_check_pdir(ioc
,"Check before sba_unmap_sg_attrs()");
1534 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
1537 while (nents
&& sglist
->dma_length
) {
1539 sba_unmap_single_attrs(dev
, sglist
->dma_address
,
1540 sglist
->dma_length
, dir
, attrs
);
1541 sglist
= sg_next(sglist
);
1545 DBG_RUN_SG("%s() DONE (nents %d)\n", __func__
, nents
);
1547 #ifdef ASSERT_PDIR_SANITY
1548 spin_lock_irqsave(&ioc
->res_lock
, flags
);
1549 sba_check_pdir(ioc
,"Check after sba_unmap_sg_attrs()");
1550 spin_unlock_irqrestore(&ioc
->res_lock
, flags
);
1554 EXPORT_SYMBOL(sba_unmap_sg_attrs
);
1556 /**************************************************************
1558 * Initialization and claim
1560 ***************************************************************/
1563 ioc_iova_init(struct ioc
*ioc
)
1567 struct pci_dev
*device
= NULL
;
1568 #ifdef FULL_VALID_PDIR
1569 unsigned long index
;
1573 ** Firmware programs the base and size of a "safe IOVA space"
1574 ** (one that doesn't overlap memory or LMMIO space) in the
1575 ** IBASE and IMASK registers.
1577 ioc
->ibase
= READ_REG(ioc
->ioc_hpa
+ IOC_IBASE
) & ~0x1UL
;
1578 ioc
->imask
= READ_REG(ioc
->ioc_hpa
+ IOC_IMASK
) | 0xFFFFFFFF00000000UL
;
1580 ioc
->iov_size
= ~ioc
->imask
+ 1;
1582 DBG_INIT("%s() hpa %p IOV base 0x%lx mask 0x%lx (%dMB)\n",
1583 __func__
, ioc
->ioc_hpa
, ioc
->ibase
, ioc
->imask
,
1584 ioc
->iov_size
>> 20);
1586 switch (iovp_size
) {
1587 case 4*1024: tcnfg
= 0; break;
1588 case 8*1024: tcnfg
= 1; break;
1589 case 16*1024: tcnfg
= 2; break;
1590 case 64*1024: tcnfg
= 3; break;
1592 panic(PFX
"Unsupported IOTLB page size %ldK",
1596 WRITE_REG(tcnfg
, ioc
->ioc_hpa
+ IOC_TCNFG
);
1598 ioc
->pdir_size
= (ioc
->iov_size
/ iovp_size
) * PDIR_ENTRY_SIZE
;
1599 ioc
->pdir_base
= (void *) __get_free_pages(GFP_KERNEL
,
1600 get_order(ioc
->pdir_size
));
1601 if (!ioc
->pdir_base
)
1602 panic(PFX
"Couldn't allocate I/O Page Table\n");
1604 memset(ioc
->pdir_base
, 0, ioc
->pdir_size
);
1606 DBG_INIT("%s() IOV page size %ldK pdir %p size %x\n", __func__
,
1607 iovp_size
>> 10, ioc
->pdir_base
, ioc
->pdir_size
);
1609 ASSERT(ALIGN((unsigned long) ioc
->pdir_base
, 4*1024) == (unsigned long) ioc
->pdir_base
);
1610 WRITE_REG(virt_to_phys(ioc
->pdir_base
), ioc
->ioc_hpa
+ IOC_PDIR_BASE
);
1613 ** If an AGP device is present, only use half of the IOV space
1614 ** for PCI DMA. Unfortunately we can't know ahead of time
1615 ** whether GART support will actually be used, for now we
1616 ** can just key on an AGP device found in the system.
1617 ** We program the next pdir index after we stop w/ a key for
1618 ** the GART code to handshake on.
1620 for_each_pci_dev(device
)
1621 agp_found
|= pci_find_capability(device
, PCI_CAP_ID_AGP
);
1623 if (agp_found
&& reserve_sba_gart
) {
1624 printk(KERN_INFO PFX
"reserving %dMb of IOVA space at 0x%lx for agpgart\n",
1625 ioc
->iov_size
/2 >> 20, ioc
->ibase
+ ioc
->iov_size
/2);
1626 ioc
->pdir_size
/= 2;
1627 ((u64
*)ioc
->pdir_base
)[PDIR_INDEX(ioc
->iov_size
/2)] = ZX1_SBA_IOMMU_COOKIE
;
1629 #ifdef FULL_VALID_PDIR
1631 ** Check to see if the spill page has been allocated, we don't need more than
1632 ** one across multiple SBAs.
1634 if (!prefetch_spill_page
) {
1635 char *spill_poison
= "SBAIOMMU POISON";
1636 int poison_size
= 16;
1637 void *poison_addr
, *addr
;
1639 addr
= (void *)__get_free_pages(GFP_KERNEL
, get_order(iovp_size
));
1641 panic(PFX
"Couldn't allocate PDIR spill page\n");
1644 for ( ; (u64
) poison_addr
< addr
+ iovp_size
; poison_addr
+= poison_size
)
1645 memcpy(poison_addr
, spill_poison
, poison_size
);
1647 prefetch_spill_page
= virt_to_phys(addr
);
1649 DBG_INIT("%s() prefetch spill addr: 0x%lx\n", __func__
, prefetch_spill_page
);
1652 ** Set all the PDIR entries valid w/ the spill page as the target
1654 for (index
= 0 ; index
< (ioc
->pdir_size
/ PDIR_ENTRY_SIZE
) ; index
++)
1655 ((u64
*)ioc
->pdir_base
)[index
] = (0x80000000000000FF | prefetch_spill_page
);
1658 /* Clear I/O TLB of any possible entries */
1659 WRITE_REG(ioc
->ibase
| (get_iovp_order(ioc
->iov_size
) + iovp_shift
), ioc
->ioc_hpa
+ IOC_PCOM
);
1660 READ_REG(ioc
->ioc_hpa
+ IOC_PCOM
);
1662 /* Enable IOVA translation */
1663 WRITE_REG(ioc
->ibase
| 1, ioc
->ioc_hpa
+ IOC_IBASE
);
1664 READ_REG(ioc
->ioc_hpa
+ IOC_IBASE
);
1668 ioc_resource_init(struct ioc
*ioc
)
1670 spin_lock_init(&ioc
->res_lock
);
1671 #if DELAYED_RESOURCE_CNT > 0
1672 spin_lock_init(&ioc
->saved_lock
);
1675 /* resource map size dictated by pdir_size */
1676 ioc
->res_size
= ioc
->pdir_size
/ PDIR_ENTRY_SIZE
; /* entries */
1677 ioc
->res_size
>>= 3; /* convert bit count to byte count */
1678 DBG_INIT("%s() res_size 0x%x\n", __func__
, ioc
->res_size
);
1680 ioc
->res_map
= (char *) __get_free_pages(GFP_KERNEL
,
1681 get_order(ioc
->res_size
));
1683 panic(PFX
"Couldn't allocate resource map\n");
1685 memset(ioc
->res_map
, 0, ioc
->res_size
);
1686 /* next available IOVP - circular search */
1687 ioc
->res_hint
= (unsigned long *) ioc
->res_map
;
1689 #ifdef ASSERT_PDIR_SANITY
1690 /* Mark first bit busy - ie no IOVA 0 */
1691 ioc
->res_map
[0] = 0x1;
1692 ioc
->pdir_base
[0] = 0x8000000000000000ULL
| ZX1_SBA_IOMMU_COOKIE
;
1694 #ifdef FULL_VALID_PDIR
1695 /* Mark the last resource used so we don't prefetch beyond IOVA space */
1696 ioc
->res_map
[ioc
->res_size
- 1] |= 0x80UL
; /* res_map is chars */
1697 ioc
->pdir_base
[(ioc
->pdir_size
/ PDIR_ENTRY_SIZE
) - 1] = (0x80000000000000FF
1698 | prefetch_spill_page
);
1701 DBG_INIT("%s() res_map %x %p\n", __func__
,
1702 ioc
->res_size
, (void *) ioc
->res_map
);
1706 ioc_sac_init(struct ioc
*ioc
)
1708 struct pci_dev
*sac
= NULL
;
1709 struct pci_controller
*controller
= NULL
;
1712 * pci_alloc_coherent() must return a DMA address which is
1713 * SAC (single address cycle) addressable, so allocate a
1714 * pseudo-device to enforce that.
1716 sac
= kzalloc(sizeof(*sac
), GFP_KERNEL
);
1718 panic(PFX
"Couldn't allocate struct pci_dev");
1720 controller
= kzalloc(sizeof(*controller
), GFP_KERNEL
);
1722 panic(PFX
"Couldn't allocate struct pci_controller");
1724 controller
->iommu
= ioc
;
1725 sac
->sysdata
= controller
;
1726 sac
->dma_mask
= 0xFFFFFFFFUL
;
1728 sac
->dev
.bus
= &pci_bus_type
;
1730 ioc
->sac_only_dev
= sac
;
1734 ioc_zx1_init(struct ioc
*ioc
)
1736 unsigned long rope_config
;
1739 if (ioc
->rev
< 0x20)
1740 panic(PFX
"IOC 2.0 or later required for IOMMU support\n");
1742 /* 38 bit memory controller + extra bit for range displaced by MMIO */
1743 ioc
->dma_mask
= (0x1UL
<< 39) - 1;
1746 ** Clear ROPE(N)_CONFIG AO bit.
1747 ** Disables "NT Ordering" (~= !"Relaxed Ordering")
1748 ** Overrides bit 1 in DMA Hint Sets.
1749 ** Improves netperf UDP_STREAM by ~10% for tg3 on bcm5701.
1751 for (i
=0; i
<(8*8); i
+=8) {
1752 rope_config
= READ_REG(ioc
->ioc_hpa
+ IOC_ROPE0_CFG
+ i
);
1753 rope_config
&= ~IOC_ROPE_AO
;
1754 WRITE_REG(rope_config
, ioc
->ioc_hpa
+ IOC_ROPE0_CFG
+ i
);
1758 typedef void (initfunc
)(struct ioc
*);
1766 static struct ioc_iommu ioc_iommu_info
[] __initdata
= {
1767 { ZX1_IOC_ID
, "zx1", ioc_zx1_init
},
1768 { ZX2_IOC_ID
, "zx2", NULL
},
1769 { SX1000_IOC_ID
, "sx1000", NULL
},
1770 { SX2000_IOC_ID
, "sx2000", NULL
},
1773 static struct ioc
* __init
1774 ioc_init(u64 hpa
, void *handle
)
1777 struct ioc_iommu
*info
;
1779 ioc
= kzalloc(sizeof(*ioc
), GFP_KERNEL
);
1783 ioc
->next
= ioc_list
;
1786 ioc
->handle
= handle
;
1787 ioc
->ioc_hpa
= ioremap(hpa
, 0x1000);
1789 ioc
->func_id
= READ_REG(ioc
->ioc_hpa
+ IOC_FUNC_ID
);
1790 ioc
->rev
= READ_REG(ioc
->ioc_hpa
+ IOC_FCLASS
) & 0xFFUL
;
1791 ioc
->dma_mask
= 0xFFFFFFFFFFFFFFFFUL
; /* conservative */
1793 for (info
= ioc_iommu_info
; info
< ioc_iommu_info
+ ARRAY_SIZE(ioc_iommu_info
); info
++) {
1794 if (ioc
->func_id
== info
->func_id
) {
1795 ioc
->name
= info
->name
;
1801 iovp_size
= (1 << iovp_shift
);
1802 iovp_mask
= ~(iovp_size
- 1);
1804 DBG_INIT("%s: PAGE_SIZE %ldK, iovp_size %ldK\n", __func__
,
1805 PAGE_SIZE
>> 10, iovp_size
>> 10);
1808 ioc
->name
= kmalloc(24, GFP_KERNEL
);
1810 sprintf((char *) ioc
->name
, "Unknown (%04x:%04x)",
1811 ioc
->func_id
& 0xFFFF, (ioc
->func_id
>> 16) & 0xFFFF);
1813 ioc
->name
= "Unknown";
1817 ioc_resource_init(ioc
);
1820 if ((long) ~iovp_mask
> (long) ia64_max_iommu_merge_mask
)
1821 ia64_max_iommu_merge_mask
= ~iovp_mask
;
1823 printk(KERN_INFO PFX
1824 "%s %d.%d HPA 0x%lx IOVA space %dMb at 0x%lx\n",
1825 ioc
->name
, (ioc
->rev
>> 4) & 0xF, ioc
->rev
& 0xF,
1826 hpa
, ioc
->iov_size
>> 20, ioc
->ibase
);
1833 /**************************************************************************
1835 ** SBA initialization code (HW and SW)
1837 ** o identify SBA chip itself
1838 ** o FIXME: initialize DMA hints for reasonable defaults
1840 **************************************************************************/
1842 #ifdef CONFIG_PROC_FS
1844 ioc_start(struct seq_file
*s
, loff_t
*pos
)
1849 for (ioc
= ioc_list
; ioc
; ioc
= ioc
->next
)
1857 ioc_next(struct seq_file
*s
, void *v
, loff_t
*pos
)
1859 struct ioc
*ioc
= v
;
1866 ioc_stop(struct seq_file
*s
, void *v
)
1871 ioc_show(struct seq_file
*s
, void *v
)
1873 struct ioc
*ioc
= v
;
1874 unsigned long *res_ptr
= (unsigned long *)ioc
->res_map
;
1877 seq_printf(s
, "Hewlett Packard %s IOC rev %d.%d\n",
1878 ioc
->name
, ((ioc
->rev
>> 4) & 0xF), (ioc
->rev
& 0xF));
1880 if (ioc
->node
!= MAX_NUMNODES
)
1881 seq_printf(s
, "NUMA node : %d\n", ioc
->node
);
1883 seq_printf(s
, "IOVA size : %ld MB\n", ((ioc
->pdir_size
>> 3) * iovp_size
)/(1024*1024));
1884 seq_printf(s
, "IOVA page size : %ld kb\n", iovp_size
/1024);
1886 for (i
= 0; i
< (ioc
->res_size
/ sizeof(unsigned long)); ++i
, ++res_ptr
)
1887 used
+= hweight64(*res_ptr
);
1889 seq_printf(s
, "PDIR size : %d entries\n", ioc
->pdir_size
>> 3);
1890 seq_printf(s
, "PDIR used : %d entries\n", used
);
1892 #ifdef PDIR_SEARCH_TIMING
1894 unsigned long i
= 0, avg
= 0, min
, max
;
1895 min
= max
= ioc
->avg_search
[0];
1896 for (i
= 0; i
< SBA_SEARCH_SAMPLE
; i
++) {
1897 avg
+= ioc
->avg_search
[i
];
1898 if (ioc
->avg_search
[i
] > max
) max
= ioc
->avg_search
[i
];
1899 if (ioc
->avg_search
[i
] < min
) min
= ioc
->avg_search
[i
];
1901 avg
/= SBA_SEARCH_SAMPLE
;
1902 seq_printf(s
, "Bitmap search : %ld/%ld/%ld (min/avg/max CPU Cycles/IOVA page)\n",
1906 #ifndef ALLOW_IOV_BYPASS
1907 seq_printf(s
, "IOVA bypass disabled\n");
1912 static const struct seq_operations ioc_seq_ops
= {
1920 ioc_open(struct inode
*inode
, struct file
*file
)
1922 return seq_open(file
, &ioc_seq_ops
);
1925 static const struct file_operations ioc_fops
= {
1928 .llseek
= seq_lseek
,
1929 .release
= seq_release
1935 struct proc_dir_entry
*dir
;
1937 dir
= proc_mkdir("bus/mckinley", NULL
);
1941 proc_create(ioc_list
->name
, 0, dir
, &ioc_fops
);
1946 sba_connect_bus(struct pci_bus
*bus
)
1948 acpi_handle handle
, parent
;
1952 if (!PCI_CONTROLLER(bus
))
1953 panic(PFX
"no sysdata on bus %d!\n", bus
->number
);
1955 if (PCI_CONTROLLER(bus
)->iommu
)
1958 handle
= PCI_CONTROLLER(bus
)->acpi_handle
;
1963 * The IOC scope encloses PCI root bridges in the ACPI
1964 * namespace, so work our way out until we find an IOC we
1965 * claimed previously.
1968 for (ioc
= ioc_list
; ioc
; ioc
= ioc
->next
)
1969 if (ioc
->handle
== handle
) {
1970 PCI_CONTROLLER(bus
)->iommu
= ioc
;
1974 status
= acpi_get_parent(handle
, &parent
);
1976 } while (ACPI_SUCCESS(status
));
1978 printk(KERN_WARNING
"No IOC for PCI Bus %04x:%02x in ACPI\n", pci_domain_nr(bus
), bus
->number
);
1983 sba_map_ioc_to_node(struct ioc
*ioc
, acpi_handle handle
)
1988 ioc
->node
= MAX_NUMNODES
;
1990 pxm
= acpi_get_pxm(handle
);
1995 node
= pxm_to_node(pxm
);
1997 if (node
>= MAX_NUMNODES
|| !node_online(node
))
2004 #define sba_map_ioc_to_node(ioc, handle)
2008 acpi_sba_ioc_add(struct acpi_device
*device
)
2013 struct acpi_buffer buffer
;
2014 struct acpi_device_info
*dev_info
;
2016 status
= hp_acpi_csr_space(device
->handle
, &hpa
, &length
);
2017 if (ACPI_FAILURE(status
))
2020 buffer
.length
= ACPI_ALLOCATE_LOCAL_BUFFER
;
2021 status
= acpi_get_object_info(device
->handle
, &buffer
);
2022 if (ACPI_FAILURE(status
))
2024 dev_info
= buffer
.pointer
;
2027 * For HWP0001, only SBA appears in ACPI namespace. It encloses the PCI
2028 * root bridges, and its CSR space includes the IOC function.
2030 if (strncmp("HWP0001", dev_info
->hardware_id
.value
, 7) == 0) {
2031 hpa
+= ZX1_IOC_OFFSET
;
2032 /* zx1 based systems default to kernel page size iommu pages */
2034 iovp_shift
= min(PAGE_SHIFT
, 16);
2039 * default anything not caught above or specified on cmdline to 4k
2045 ioc
= ioc_init(hpa
, device
->handle
);
2049 /* setup NUMA node association */
2050 sba_map_ioc_to_node(ioc
, device
->handle
);
2054 static const struct acpi_device_id hp_ioc_iommu_device_ids
[] = {
2059 static struct acpi_driver acpi_sba_ioc_driver
= {
2060 .name
= "IOC IOMMU Driver",
2061 .ids
= hp_ioc_iommu_device_ids
,
2063 .add
= acpi_sba_ioc_add
,
2070 if (!ia64_platform_is("hpzx1") && !ia64_platform_is("hpzx1_swiotlb"))
2073 #if defined(CONFIG_IA64_GENERIC) && defined(CONFIG_CRASH_DUMP) && \
2074 defined(CONFIG_PROC_FS)
2075 /* If we are booting a kdump kernel, the sba_iommu will
2076 * cause devices that were not shutdown properly to MCA
2077 * as soon as they are turned back on. Our only option for
2078 * a successful kdump kernel boot is to use the swiotlb.
2080 if (elfcorehdr_addr
< ELFCORE_ADDR_MAX
) {
2081 if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
2082 panic("Unable to initialize software I/O TLB:"
2083 " Try machvec=dig boot option");
2084 machvec_init("dig");
2089 acpi_bus_register_driver(&acpi_sba_ioc_driver
);
2091 #ifdef CONFIG_IA64_GENERIC
2093 * If we didn't find something sba_iommu can claim, we
2094 * need to setup the swiotlb and switch to the dig machvec.
2096 if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
2097 panic("Unable to find SBA IOMMU or initialize "
2098 "software I/O TLB: Try machvec=dig boot option");
2099 machvec_init("dig");
2101 panic("Unable to find SBA IOMMU: Try a generic or DIG kernel");
2106 #if defined(CONFIG_IA64_GENERIC) || defined(CONFIG_IA64_HP_ZX1_SWIOTLB)
2108 * hpzx1_swiotlb needs to have a fairly small swiotlb bounce
2109 * buffer setup to support devices with smaller DMA masks than
2110 * sba_iommu can handle.
2112 if (ia64_platform_is("hpzx1_swiotlb")) {
2113 extern void hwsw_init(void);
2121 struct pci_bus
*b
= NULL
;
2122 while ((b
= pci_find_next_bus(b
)) != NULL
)
2127 #ifdef CONFIG_PROC_FS
2133 subsys_initcall(sba_init
); /* must be initialized after ACPI etc., but before any drivers... */
2136 nosbagart(char *str
)
2138 reserve_sba_gart
= 0;
2143 sba_dma_supported (struct device
*dev
, u64 mask
)
2145 /* make sure it's at least 32bit capable */
2146 return ((mask
& 0xFFFFFFFFUL
) == 0xFFFFFFFFUL
);
2150 sba_dma_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
2155 __setup("nosbagart", nosbagart
);
2158 sba_page_override(char *str
)
2160 unsigned long page_size
;
2162 page_size
= memparse(str
, &str
);
2163 switch (page_size
) {
2168 iovp_shift
= ffs(page_size
) - 1;
2171 printk("%s: unknown/unsupported iommu page size %ld\n",
2172 __func__
, page_size
);
2178 __setup("sbapagesize=",sba_page_override
);
2180 EXPORT_SYMBOL(sba_dma_mapping_error
);
2181 EXPORT_SYMBOL(sba_dma_supported
);
2182 EXPORT_SYMBOL(sba_alloc_coherent
);
2183 EXPORT_SYMBOL(sba_free_coherent
);