2 * linux/arch/arm/mm/dma-mapping.c
4 * Copyright (C) 2000-2004 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
10 * DMA uncached mapping support.
12 #include <linux/module.h>
14 #include <linux/gfp.h>
15 #include <linux/errno.h>
16 #include <linux/list.h>
17 #include <linux/init.h>
18 #include <linux/device.h>
19 #include <linux/dma-mapping.h>
21 #include <asm/memory.h>
22 #include <asm/highmem.h>
23 #include <asm/cacheflush.h>
24 #include <asm/tlbflush.h>
25 #include <asm/sizes.h>
27 static u64
get_coherent_dma_mask(struct device
*dev
)
29 u64 mask
= ISA_DMA_THRESHOLD
;
32 mask
= dev
->coherent_dma_mask
;
35 * Sanity check the DMA mask - it must be non-zero, and
36 * must be able to be satisfied by a DMA allocation.
39 dev_warn(dev
, "coherent DMA mask is unset\n");
43 if ((~mask
) & ISA_DMA_THRESHOLD
) {
44 dev_warn(dev
, "coherent DMA mask %#llx is smaller "
45 "than system GFP_DMA mask %#llx\n",
46 mask
, (unsigned long long)ISA_DMA_THRESHOLD
);
55 * Allocate a DMA buffer for 'dev' of size 'size' using the
56 * specified gfp mask. Note that 'size' must be page aligned.
58 static struct page
*__dma_alloc_buffer(struct device
*dev
, size_t size
, gfp_t gfp
)
60 unsigned long order
= get_order(size
);
61 struct page
*page
, *p
, *e
;
63 u64 mask
= get_coherent_dma_mask(dev
);
65 #ifdef CONFIG_DMA_API_DEBUG
66 u64 limit
= (mask
+ 1) & ~mask
;
67 if (limit
&& size
>= limit
) {
68 dev_warn(dev
, "coherent allocation too big (requested %#x mask %#llx)\n",
77 if (mask
< 0xffffffffULL
)
80 page
= alloc_pages(gfp
, order
);
85 * Now split the huge page and free the excess pages
87 split_page(page
, order
);
88 for (p
= page
+ (size
>> PAGE_SHIFT
), e
= page
+ (1 << order
); p
< e
; p
++)
92 * Ensure that the allocated pages are zeroed, and that any data
93 * lurking in the kernel direct-mapped region is invalidated.
95 ptr
= page_address(page
);
97 dmac_flush_range(ptr
, ptr
+ size
);
98 outer_flush_range(__pa(ptr
), __pa(ptr
) + size
);
104 * Free a DMA buffer. 'size' must be page aligned.
106 static void __dma_free_buffer(struct page
*page
, size_t size
)
108 struct page
*e
= page
+ (size
>> PAGE_SHIFT
);
117 /* Sanity check size */
118 #if (CONSISTENT_DMA_SIZE % SZ_2M)
119 #error "CONSISTENT_DMA_SIZE must be multiple of 2MiB"
122 #define CONSISTENT_OFFSET(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PAGE_SHIFT)
123 #define CONSISTENT_PTE_INDEX(x) (((unsigned long)(x) - CONSISTENT_BASE) >> PGDIR_SHIFT)
124 #define NUM_CONSISTENT_PTES (CONSISTENT_DMA_SIZE >> PGDIR_SHIFT)
127 * These are the page tables (2MB each) covering uncached, DMA consistent allocations
129 static pte_t
*consistent_pte
[NUM_CONSISTENT_PTES
];
131 #include "vmregion.h"
133 static struct arm_vmregion_head consistent_head
= {
134 .vm_lock
= __SPIN_LOCK_UNLOCKED(&consistent_head
.vm_lock
),
135 .vm_list
= LIST_HEAD_INIT(consistent_head
.vm_list
),
136 .vm_start
= CONSISTENT_BASE
,
137 .vm_end
= CONSISTENT_END
,
140 #ifdef CONFIG_HUGETLB_PAGE
141 #error ARM Coherent DMA allocator does not (yet) support huge TLB
145 * Initialise the consistent memory allocation.
147 static int __init
consistent_init(void)
154 u32 base
= CONSISTENT_BASE
;
157 pgd
= pgd_offset(&init_mm
, base
);
158 pmd
= pmd_alloc(&init_mm
, pgd
, base
);
160 printk(KERN_ERR
"%s: no pmd tables\n", __func__
);
164 WARN_ON(!pmd_none(*pmd
));
166 pte
= pte_alloc_kernel(pmd
, base
);
168 printk(KERN_ERR
"%s: no pte tables\n", __func__
);
173 consistent_pte
[i
++] = pte
;
174 base
+= (1 << PGDIR_SHIFT
);
175 } while (base
< CONSISTENT_END
);
180 core_initcall(consistent_init
);
183 __dma_alloc_remap(struct page
*page
, size_t size
, gfp_t gfp
, pgprot_t prot
)
185 struct arm_vmregion
*c
;
189 if (!consistent_pte
[0]) {
190 printk(KERN_ERR
"%s: not initialised\n", __func__
);
196 * Align the virtual region allocation - maximum alignment is
197 * a section size, minimum is a page size. This helps reduce
198 * fragmentation of the DMA space, and also prevents allocations
199 * smaller than a section from crossing a section boundary.
201 bit
= fls(size
- 1) + 1;
202 if (bit
> SECTION_SHIFT
)
207 * Allocate a virtual address in the consistent mapping region.
209 c
= arm_vmregion_alloc(&consistent_head
, align
, size
,
210 gfp
& ~(__GFP_DMA
| __GFP_HIGHMEM
));
213 int idx
= CONSISTENT_PTE_INDEX(c
->vm_start
);
214 u32 off
= CONSISTENT_OFFSET(c
->vm_start
) & (PTRS_PER_PTE
-1);
216 pte
= consistent_pte
[idx
] + off
;
220 BUG_ON(!pte_none(*pte
));
222 set_pte_ext(pte
, mk_pte(page
, prot
), 0);
226 if (off
>= PTRS_PER_PTE
) {
228 pte
= consistent_pte
[++idx
];
230 } while (size
-= PAGE_SIZE
);
232 return (void *)c
->vm_start
;
237 static void __dma_free_remap(void *cpu_addr
, size_t size
)
239 struct arm_vmregion
*c
;
245 c
= arm_vmregion_find_remove(&consistent_head
, (unsigned long)cpu_addr
);
247 printk(KERN_ERR
"%s: trying to free invalid coherent area: %p\n",
253 if ((c
->vm_end
- c
->vm_start
) != size
) {
254 printk(KERN_ERR
"%s: freeing wrong coherent size (%ld != %d)\n",
255 __func__
, c
->vm_end
- c
->vm_start
, size
);
257 size
= c
->vm_end
- c
->vm_start
;
260 idx
= CONSISTENT_PTE_INDEX(c
->vm_start
);
261 off
= CONSISTENT_OFFSET(c
->vm_start
) & (PTRS_PER_PTE
-1);
262 ptep
= consistent_pte
[idx
] + off
;
265 pte_t pte
= ptep_get_and_clear(&init_mm
, addr
, ptep
);
270 if (off
>= PTRS_PER_PTE
) {
272 ptep
= consistent_pte
[++idx
];
275 if (pte_none(pte
) || !pte_present(pte
))
276 printk(KERN_CRIT
"%s: bad page in kernel page table\n",
278 } while (size
-= PAGE_SIZE
);
280 flush_tlb_kernel_range(c
->vm_start
, c
->vm_end
);
282 arm_vmregion_free(&consistent_head
, c
);
285 #else /* !CONFIG_MMU */
287 #define __dma_alloc_remap(page, size, gfp, prot) page_address(page)
288 #define __dma_free_remap(addr, size) do { } while (0)
290 #endif /* CONFIG_MMU */
293 __dma_alloc(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
,
300 size
= PAGE_ALIGN(size
);
302 page
= __dma_alloc_buffer(dev
, size
, gfp
);
306 if (!arch_is_coherent())
307 addr
= __dma_alloc_remap(page
, size
, gfp
, prot
);
309 addr
= page_address(page
);
312 *handle
= page_to_dma(dev
, page
);
318 * Allocate DMA-coherent memory space and return both the kernel remapped
319 * virtual and bus address for that space.
322 dma_alloc_coherent(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
)
326 if (dma_alloc_from_coherent(dev
, size
, handle
, &memory
))
329 return __dma_alloc(dev
, size
, handle
, gfp
,
330 pgprot_dmacoherent(pgprot_kernel
));
332 EXPORT_SYMBOL(dma_alloc_coherent
);
335 * Allocate a writecombining region, in much the same way as
336 * dma_alloc_coherent above.
339 dma_alloc_writecombine(struct device
*dev
, size_t size
, dma_addr_t
*handle
, gfp_t gfp
)
341 return __dma_alloc(dev
, size
, handle
, gfp
,
342 pgprot_writecombine(pgprot_kernel
));
344 EXPORT_SYMBOL(dma_alloc_writecombine
);
346 static int dma_mmap(struct device
*dev
, struct vm_area_struct
*vma
,
347 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
351 unsigned long user_size
, kern_size
;
352 struct arm_vmregion
*c
;
354 user_size
= (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
;
356 c
= arm_vmregion_find(&consistent_head
, (unsigned long)cpu_addr
);
358 unsigned long off
= vma
->vm_pgoff
;
360 kern_size
= (c
->vm_end
- c
->vm_start
) >> PAGE_SHIFT
;
362 if (off
< kern_size
&&
363 user_size
<= (kern_size
- off
)) {
364 ret
= remap_pfn_range(vma
, vma
->vm_start
,
365 page_to_pfn(c
->vm_pages
) + off
,
366 user_size
<< PAGE_SHIFT
,
370 #endif /* CONFIG_MMU */
375 int dma_mmap_coherent(struct device
*dev
, struct vm_area_struct
*vma
,
376 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
378 vma
->vm_page_prot
= pgprot_dmacoherent(vma
->vm_page_prot
);
379 return dma_mmap(dev
, vma
, cpu_addr
, dma_addr
, size
);
381 EXPORT_SYMBOL(dma_mmap_coherent
);
383 int dma_mmap_writecombine(struct device
*dev
, struct vm_area_struct
*vma
,
384 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
)
386 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
387 return dma_mmap(dev
, vma
, cpu_addr
, dma_addr
, size
);
389 EXPORT_SYMBOL(dma_mmap_writecombine
);
392 * free a page as defined by the above mapping.
393 * Must not be called with IRQs disabled.
395 void dma_free_coherent(struct device
*dev
, size_t size
, void *cpu_addr
, dma_addr_t handle
)
397 WARN_ON(irqs_disabled());
399 if (dma_release_from_coherent(dev
, get_order(size
), cpu_addr
))
402 size
= PAGE_ALIGN(size
);
404 if (!arch_is_coherent())
405 __dma_free_remap(cpu_addr
, size
);
407 __dma_free_buffer(dma_to_page(dev
, handle
), size
);
409 EXPORT_SYMBOL(dma_free_coherent
);
412 * Make an area consistent for devices.
413 * Note: Drivers should NOT use this function directly, as it will break
414 * platforms with CONFIG_DMABOUNCE.
415 * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
417 void ___dma_single_cpu_to_dev(const void *kaddr
, size_t size
,
418 enum dma_data_direction dir
)
422 BUG_ON(!virt_addr_valid(kaddr
) || !virt_addr_valid(kaddr
+ size
- 1));
424 dmac_map_area(kaddr
, size
, dir
);
427 if (dir
== DMA_FROM_DEVICE
) {
428 outer_inv_range(paddr
, paddr
+ size
);
430 outer_clean_range(paddr
, paddr
+ size
);
432 /* FIXME: non-speculating: flush on bidirectional mappings? */
434 EXPORT_SYMBOL(___dma_single_cpu_to_dev
);
436 void ___dma_single_dev_to_cpu(const void *kaddr
, size_t size
,
437 enum dma_data_direction dir
)
439 BUG_ON(!virt_addr_valid(kaddr
) || !virt_addr_valid(kaddr
+ size
- 1));
441 /* FIXME: non-speculating: not required */
442 /* don't bother invalidating if DMA to device */
443 if (dir
!= DMA_TO_DEVICE
) {
444 unsigned long paddr
= __pa(kaddr
);
445 outer_inv_range(paddr
, paddr
+ size
);
448 dmac_unmap_area(kaddr
, size
, dir
);
450 EXPORT_SYMBOL(___dma_single_dev_to_cpu
);
452 static void dma_cache_maint_page(struct page
*page
, unsigned long offset
,
453 size_t size
, enum dma_data_direction dir
,
454 void (*op
)(const void *, size_t, int))
457 * A single sg entry may refer to multiple physically contiguous
458 * pages. But we still need to process highmem pages individually.
459 * If highmem is not configured then the bulk of this loop gets
467 if (PageHighMem(page
)) {
468 if (len
+ offset
> PAGE_SIZE
) {
469 if (offset
>= PAGE_SIZE
) {
470 page
+= offset
/ PAGE_SIZE
;
473 len
= PAGE_SIZE
- offset
;
475 vaddr
= kmap_high_get(page
);
480 } else if (cache_is_vipt()) {
482 vaddr
= kmap_high_l1_vipt(page
, &saved_pte
);
483 op(vaddr
+ offset
, len
, dir
);
484 kunmap_high_l1_vipt(page
, saved_pte
);
487 vaddr
= page_address(page
) + offset
;
496 void ___dma_page_cpu_to_dev(struct page
*page
, unsigned long off
,
497 size_t size
, enum dma_data_direction dir
)
501 dma_cache_maint_page(page
, off
, size
, dir
, dmac_map_area
);
503 paddr
= page_to_phys(page
) + off
;
504 if (dir
== DMA_FROM_DEVICE
) {
505 outer_inv_range(paddr
, paddr
+ size
);
507 outer_clean_range(paddr
, paddr
+ size
);
509 /* FIXME: non-speculating: flush on bidirectional mappings? */
511 EXPORT_SYMBOL(___dma_page_cpu_to_dev
);
513 void ___dma_page_dev_to_cpu(struct page
*page
, unsigned long off
,
514 size_t size
, enum dma_data_direction dir
)
516 unsigned long paddr
= page_to_phys(page
) + off
;
518 /* FIXME: non-speculating: not required */
519 /* don't bother invalidating if DMA to device */
520 if (dir
!= DMA_TO_DEVICE
)
521 outer_inv_range(paddr
, paddr
+ size
);
523 dma_cache_maint_page(page
, off
, size
, dir
, dmac_unmap_area
);
525 EXPORT_SYMBOL(___dma_page_dev_to_cpu
);
528 * dma_map_sg - map a set of SG buffers for streaming mode DMA
529 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
530 * @sg: list of buffers
531 * @nents: number of buffers to map
532 * @dir: DMA transfer direction
534 * Map a set of buffers described by scatterlist in streaming mode for DMA.
535 * This is the scatter-gather version of the dma_map_single interface.
536 * Here the scatter gather list elements are each tagged with the
537 * appropriate dma address and length. They are obtained via
538 * sg_dma_{address,length}.
540 * Device ownership issues as mentioned for dma_map_single are the same
543 int dma_map_sg(struct device
*dev
, struct scatterlist
*sg
, int nents
,
544 enum dma_data_direction dir
)
546 struct scatterlist
*s
;
549 for_each_sg(sg
, s
, nents
, i
) {
550 s
->dma_address
= dma_map_page(dev
, sg_page(s
), s
->offset
,
552 if (dma_mapping_error(dev
, s
->dma_address
))
558 for_each_sg(sg
, s
, i
, j
)
559 dma_unmap_page(dev
, sg_dma_address(s
), sg_dma_len(s
), dir
);
562 EXPORT_SYMBOL(dma_map_sg
);
565 * dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg
566 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
567 * @sg: list of buffers
568 * @nents: number of buffers to unmap (returned from dma_map_sg)
569 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
571 * Unmap a set of streaming mode DMA translations. Again, CPU access
572 * rules concerning calls here are the same as for dma_unmap_single().
574 void dma_unmap_sg(struct device
*dev
, struct scatterlist
*sg
, int nents
,
575 enum dma_data_direction dir
)
577 struct scatterlist
*s
;
580 for_each_sg(sg
, s
, nents
, i
)
581 dma_unmap_page(dev
, sg_dma_address(s
), sg_dma_len(s
), dir
);
583 EXPORT_SYMBOL(dma_unmap_sg
);
586 * dma_sync_sg_for_cpu
587 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
588 * @sg: list of buffers
589 * @nents: number of buffers to map (returned from dma_map_sg)
590 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
592 void dma_sync_sg_for_cpu(struct device
*dev
, struct scatterlist
*sg
,
593 int nents
, enum dma_data_direction dir
)
595 struct scatterlist
*s
;
598 for_each_sg(sg
, s
, nents
, i
) {
599 if (!dmabounce_sync_for_cpu(dev
, sg_dma_address(s
), 0,
603 __dma_page_dev_to_cpu(sg_page(s
), s
->offset
,
607 EXPORT_SYMBOL(dma_sync_sg_for_cpu
);
610 * dma_sync_sg_for_device
611 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
612 * @sg: list of buffers
613 * @nents: number of buffers to map (returned from dma_map_sg)
614 * @dir: DMA transfer direction (same as was passed to dma_map_sg)
616 void dma_sync_sg_for_device(struct device
*dev
, struct scatterlist
*sg
,
617 int nents
, enum dma_data_direction dir
)
619 struct scatterlist
*s
;
622 for_each_sg(sg
, s
, nents
, i
) {
623 if (!dmabounce_sync_for_device(dev
, sg_dma_address(s
), 0,
627 __dma_page_cpu_to_dev(sg_page(s
), s
->offset
,
631 EXPORT_SYMBOL(dma_sync_sg_for_device
);