2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
11 #include <linux/types.h>
12 #include <linux/dma-mapping.h>
14 #include <linux/export.h>
15 #include <linux/scatterlist.h>
16 #include <linux/string.h>
17 #include <linux/gfp.h>
18 #include <linux/highmem.h>
19 #include <linux/dma-contiguous.h>
21 #include <asm/cache.h>
22 #include <asm/cpu-type.h>
25 #include <dma-coherence.h>
27 #if defined(CONFIG_DMA_MAYBE_COHERENT) && !defined(CONFIG_DMA_PERDEV_COHERENT)
28 /* User defined DMA coherency from command line. */
29 enum coherent_io_user_state coherentio
= IO_COHERENCE_DEFAULT
;
30 EXPORT_SYMBOL_GPL(coherentio
);
31 int hw_coherentio
= 0; /* Actual hardware supported DMA coherency setting. */
33 static int __init
setcoherentio(char *str
)
35 coherentio
= IO_COHERENCE_ENABLED
;
36 pr_info("Hardware DMA cache coherency (command line)\n");
39 early_param("coherentio", setcoherentio
);
41 static int __init
setnocoherentio(char *str
)
43 coherentio
= IO_COHERENCE_DISABLED
;
44 pr_info("Software DMA cache coherency (command line)\n");
47 early_param("nocoherentio", setnocoherentio
);
50 static inline struct page
*dma_addr_to_page(struct device
*dev
,
54 plat_dma_addr_to_phys(dev
, dma_addr
) >> PAGE_SHIFT
);
58 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
59 * speculatively fill random cachelines with stale data at any time,
60 * requiring an extra flush post-DMA.
62 * Warning on the terminology - Linux calls an uncached area coherent;
63 * MIPS terminology calls memory areas with hardware maintained coherency
66 * Note that the R14000 and R16000 should also be checked for in this
67 * condition. However this function is only called on non-I/O-coherent
68 * systems and only the R10000 and R12000 are used in such systems, the
69 * SGI IP28 Indigo² rsp. SGI IP32 aka O2.
71 static inline bool cpu_needs_post_dma_flush(struct device
*dev
)
73 if (plat_device_is_coherent(dev
))
76 switch (boot_cpu_type()) {
84 * Presence of MAARs suggests that the CPU supports
85 * speculatively prefetching data, and therefore requires
86 * the post-DMA flush/invalidate.
92 static gfp_t
massage_gfp_flags(const struct device
*dev
, gfp_t gfp
)
96 /* ignore region specifiers */
97 gfp
&= ~(__GFP_DMA
| __GFP_DMA32
| __GFP_HIGHMEM
);
101 dma_flag
= __GFP_DMA
;
104 #if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
105 if (dev
== NULL
|| dev
->coherent_dma_mask
< DMA_BIT_MASK(32))
106 dma_flag
= __GFP_DMA
;
107 else if (dev
->coherent_dma_mask
< DMA_BIT_MASK(64))
108 dma_flag
= __GFP_DMA32
;
111 #if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
112 if (dev
== NULL
|| dev
->coherent_dma_mask
< DMA_BIT_MASK(64))
113 dma_flag
= __GFP_DMA32
;
116 #if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
118 dev
->coherent_dma_mask
< DMA_BIT_MASK(sizeof(phys_addr_t
) * 8))
119 dma_flag
= __GFP_DMA
;
124 /* Don't invoke OOM killer */
125 gfp
|= __GFP_NORETRY
;
127 return gfp
| dma_flag
;
130 static void *mips_dma_alloc_noncoherent(struct device
*dev
, size_t size
,
131 dma_addr_t
* dma_handle
, gfp_t gfp
)
135 gfp
= massage_gfp_flags(dev
, gfp
);
137 ret
= (void *) __get_free_pages(gfp
, get_order(size
));
140 memset(ret
, 0, size
);
141 *dma_handle
= plat_map_dma_mem(dev
, ret
, size
);
147 static void *mips_dma_alloc_coherent(struct device
*dev
, size_t size
,
148 dma_addr_t
*dma_handle
, gfp_t gfp
, unsigned long attrs
)
151 struct page
*page
= NULL
;
152 unsigned int count
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
155 * XXX: seems like the coherent and non-coherent implementations could
158 if (attrs
& DMA_ATTR_NON_CONSISTENT
)
159 return mips_dma_alloc_noncoherent(dev
, size
, dma_handle
, gfp
);
161 gfp
= massage_gfp_flags(dev
, gfp
);
163 if (IS_ENABLED(CONFIG_DMA_CMA
) && gfpflags_allow_blocking(gfp
))
164 page
= dma_alloc_from_contiguous(dev
, count
, get_order(size
),
167 page
= alloc_pages(gfp
, get_order(size
));
172 ret
= page_address(page
);
173 memset(ret
, 0, size
);
174 *dma_handle
= plat_map_dma_mem(dev
, ret
, size
);
175 if (!plat_device_is_coherent(dev
)) {
176 dma_cache_wback_inv((unsigned long) ret
, size
);
177 ret
= UNCAC_ADDR(ret
);
184 static void mips_dma_free_noncoherent(struct device
*dev
, size_t size
,
185 void *vaddr
, dma_addr_t dma_handle
)
187 plat_unmap_dma_mem(dev
, dma_handle
, size
, DMA_BIDIRECTIONAL
);
188 free_pages((unsigned long) vaddr
, get_order(size
));
191 static void mips_dma_free_coherent(struct device
*dev
, size_t size
, void *vaddr
,
192 dma_addr_t dma_handle
, unsigned long attrs
)
194 unsigned long addr
= (unsigned long) vaddr
;
195 unsigned int count
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
196 struct page
*page
= NULL
;
198 if (attrs
& DMA_ATTR_NON_CONSISTENT
) {
199 mips_dma_free_noncoherent(dev
, size
, vaddr
, dma_handle
);
203 plat_unmap_dma_mem(dev
, dma_handle
, size
, DMA_BIDIRECTIONAL
);
205 if (!plat_device_is_coherent(dev
))
206 addr
= CAC_ADDR(addr
);
208 page
= virt_to_page((void *) addr
);
210 if (!dma_release_from_contiguous(dev
, page
, count
))
211 __free_pages(page
, get_order(size
));
214 static int mips_dma_mmap(struct device
*dev
, struct vm_area_struct
*vma
,
215 void *cpu_addr
, dma_addr_t dma_addr
, size_t size
,
218 unsigned long user_count
= (vma
->vm_end
- vma
->vm_start
) >> PAGE_SHIFT
;
219 unsigned long count
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
220 unsigned long addr
= (unsigned long)cpu_addr
;
221 unsigned long off
= vma
->vm_pgoff
;
225 if (!plat_device_is_coherent(dev
))
226 addr
= CAC_ADDR(addr
);
228 pfn
= page_to_pfn(virt_to_page((void *)addr
));
230 if (attrs
& DMA_ATTR_WRITE_COMBINE
)
231 vma
->vm_page_prot
= pgprot_writecombine(vma
->vm_page_prot
);
233 vma
->vm_page_prot
= pgprot_noncached(vma
->vm_page_prot
);
235 if (dma_mmap_from_dev_coherent(dev
, vma
, cpu_addr
, size
, &ret
))
238 if (off
< count
&& user_count
<= (count
- off
)) {
239 ret
= remap_pfn_range(vma
, vma
->vm_start
,
241 user_count
<< PAGE_SHIFT
,
248 static inline void __dma_sync_virtual(void *addr
, size_t size
,
249 enum dma_data_direction direction
)
253 dma_cache_wback((unsigned long)addr
, size
);
256 case DMA_FROM_DEVICE
:
257 dma_cache_inv((unsigned long)addr
, size
);
260 case DMA_BIDIRECTIONAL
:
261 dma_cache_wback_inv((unsigned long)addr
, size
);
270 * A single sg entry may refer to multiple physically contiguous
271 * pages. But we still need to process highmem pages individually.
272 * If highmem is not configured then the bulk of this loop gets
275 static inline void __dma_sync(struct page
*page
,
276 unsigned long offset
, size_t size
, enum dma_data_direction direction
)
283 if (PageHighMem(page
)) {
286 if (offset
+ len
> PAGE_SIZE
) {
287 if (offset
>= PAGE_SIZE
) {
288 page
+= offset
>> PAGE_SHIFT
;
289 offset
&= ~PAGE_MASK
;
291 len
= PAGE_SIZE
- offset
;
294 addr
= kmap_atomic(page
);
295 __dma_sync_virtual(addr
+ offset
, len
, direction
);
298 __dma_sync_virtual(page_address(page
) + offset
,
306 static void mips_dma_unmap_page(struct device
*dev
, dma_addr_t dma_addr
,
307 size_t size
, enum dma_data_direction direction
, unsigned long attrs
)
309 if (cpu_needs_post_dma_flush(dev
) && !(attrs
& DMA_ATTR_SKIP_CPU_SYNC
))
310 __dma_sync(dma_addr_to_page(dev
, dma_addr
),
311 dma_addr
& ~PAGE_MASK
, size
, direction
);
312 plat_post_dma_flush(dev
);
313 plat_unmap_dma_mem(dev
, dma_addr
, size
, direction
);
316 static int mips_dma_map_sg(struct device
*dev
, struct scatterlist
*sglist
,
317 int nents
, enum dma_data_direction direction
, unsigned long attrs
)
320 struct scatterlist
*sg
;
322 for_each_sg(sglist
, sg
, nents
, i
) {
323 if (!plat_device_is_coherent(dev
) &&
324 !(attrs
& DMA_ATTR_SKIP_CPU_SYNC
))
325 __dma_sync(sg_page(sg
), sg
->offset
, sg
->length
,
327 #ifdef CONFIG_NEED_SG_DMA_LENGTH
328 sg
->dma_length
= sg
->length
;
330 sg
->dma_address
= plat_map_dma_mem_page(dev
, sg_page(sg
)) +
337 static dma_addr_t
mips_dma_map_page(struct device
*dev
, struct page
*page
,
338 unsigned long offset
, size_t size
, enum dma_data_direction direction
,
341 if (!plat_device_is_coherent(dev
) && !(attrs
& DMA_ATTR_SKIP_CPU_SYNC
))
342 __dma_sync(page
, offset
, size
, direction
);
344 return plat_map_dma_mem_page(dev
, page
) + offset
;
347 static void mips_dma_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
348 int nhwentries
, enum dma_data_direction direction
,
352 struct scatterlist
*sg
;
354 for_each_sg(sglist
, sg
, nhwentries
, i
) {
355 if (!plat_device_is_coherent(dev
) &&
356 !(attrs
& DMA_ATTR_SKIP_CPU_SYNC
) &&
357 direction
!= DMA_TO_DEVICE
)
358 __dma_sync(sg_page(sg
), sg
->offset
, sg
->length
,
360 plat_unmap_dma_mem(dev
, sg
->dma_address
, sg
->length
, direction
);
364 static void mips_dma_sync_single_for_cpu(struct device
*dev
,
365 dma_addr_t dma_handle
, size_t size
, enum dma_data_direction direction
)
367 if (cpu_needs_post_dma_flush(dev
))
368 __dma_sync(dma_addr_to_page(dev
, dma_handle
),
369 dma_handle
& ~PAGE_MASK
, size
, direction
);
370 plat_post_dma_flush(dev
);
373 static void mips_dma_sync_single_for_device(struct device
*dev
,
374 dma_addr_t dma_handle
, size_t size
, enum dma_data_direction direction
)
376 if (!plat_device_is_coherent(dev
))
377 __dma_sync(dma_addr_to_page(dev
, dma_handle
),
378 dma_handle
& ~PAGE_MASK
, size
, direction
);
381 static void mips_dma_sync_sg_for_cpu(struct device
*dev
,
382 struct scatterlist
*sglist
, int nelems
,
383 enum dma_data_direction direction
)
386 struct scatterlist
*sg
;
388 if (cpu_needs_post_dma_flush(dev
)) {
389 for_each_sg(sglist
, sg
, nelems
, i
) {
390 __dma_sync(sg_page(sg
), sg
->offset
, sg
->length
,
394 plat_post_dma_flush(dev
);
397 static void mips_dma_sync_sg_for_device(struct device
*dev
,
398 struct scatterlist
*sglist
, int nelems
,
399 enum dma_data_direction direction
)
402 struct scatterlist
*sg
;
404 if (!plat_device_is_coherent(dev
)) {
405 for_each_sg(sglist
, sg
, nelems
, i
) {
406 __dma_sync(sg_page(sg
), sg
->offset
, sg
->length
,
412 int mips_dma_mapping_error(struct device
*dev
, dma_addr_t dma_addr
)
417 int mips_dma_supported(struct device
*dev
, u64 mask
)
419 return plat_dma_supported(dev
, mask
);
422 void dma_cache_sync(struct device
*dev
, void *vaddr
, size_t size
,
423 enum dma_data_direction direction
)
425 BUG_ON(direction
== DMA_NONE
);
427 if (!plat_device_is_coherent(dev
))
428 __dma_sync_virtual(vaddr
, size
, direction
);
431 EXPORT_SYMBOL(dma_cache_sync
);
433 static const struct dma_map_ops mips_default_dma_map_ops
= {
434 .alloc
= mips_dma_alloc_coherent
,
435 .free
= mips_dma_free_coherent
,
436 .mmap
= mips_dma_mmap
,
437 .map_page
= mips_dma_map_page
,
438 .unmap_page
= mips_dma_unmap_page
,
439 .map_sg
= mips_dma_map_sg
,
440 .unmap_sg
= mips_dma_unmap_sg
,
441 .sync_single_for_cpu
= mips_dma_sync_single_for_cpu
,
442 .sync_single_for_device
= mips_dma_sync_single_for_device
,
443 .sync_sg_for_cpu
= mips_dma_sync_sg_for_cpu
,
444 .sync_sg_for_device
= mips_dma_sync_sg_for_device
,
445 .mapping_error
= mips_dma_mapping_error
,
446 .dma_supported
= mips_dma_supported
449 const struct dma_map_ops
*mips_dma_map_ops
= &mips_default_dma_map_ops
;
450 EXPORT_SYMBOL(mips_dma_map_ops
);
452 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
454 static int __init
mips_dma_init(void)
456 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
460 fs_initcall(mips_dma_init
);