2 * Based on linux/arch/arm/mm/dma-mapping.c
4 * Copyright (C) 2000-2004 Russell King
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
12 #include <linux/export.h>
14 #include <linux/dma-mapping.h>
15 #include <linux/scatterlist.h>
17 #include <asm/cachetype.h>
18 #include <asm/cacheflush.h>
19 #include <asm/outercache.h>
25 * dma_noop_ops is used if
27 * - cpu is v7m w/o cache support
28 * - device is coherent
29 * otherwise arm_nommu_dma_ops is used.
31 * arm_nommu_dma_ops rely on consistent DMA memory (please, refer to
32 * [1] on how to declare such memory).
34 * [1] Documentation/devicetree/bindings/reserved-memory/reserved-memory.txt
37 static void *arm_nommu_dma_alloc(struct device
*dev
, size_t size
,
38 dma_addr_t
*dma_handle
, gfp_t gfp
,
42 const struct dma_map_ops
*ops
= &dma_noop_ops
;
45 * We are here because:
46 * - no consistent DMA region has been defined, so we can't
48 * - there is no space left in consistent DMA region, so we
49 * only can fallback to generic allocator if we are
50 * advertised that consistency is not required.
53 if (attrs
& DMA_ATTR_NON_CONSISTENT
)
54 return ops
->alloc(dev
, size
, dma_handle
, gfp
, attrs
);
60 static void arm_nommu_dma_free(struct device
*dev
, size_t size
,
61 void *cpu_addr
, dma_addr_t dma_addr
,
64 const struct dma_map_ops
*ops
= &dma_noop_ops
;
66 if (attrs
& DMA_ATTR_NON_CONSISTENT
)
67 ops
->free(dev
, size
, cpu_addr
, dma_addr
, attrs
);
74 static void __dma_page_cpu_to_dev(phys_addr_t paddr
, size_t size
,
75 enum dma_data_direction dir
)
77 dmac_map_area(__va(paddr
), size
, dir
);
79 if (dir
== DMA_FROM_DEVICE
)
80 outer_inv_range(paddr
, paddr
+ size
);
82 outer_clean_range(paddr
, paddr
+ size
);
85 static void __dma_page_dev_to_cpu(phys_addr_t paddr
, size_t size
,
86 enum dma_data_direction dir
)
88 if (dir
!= DMA_TO_DEVICE
) {
89 outer_inv_range(paddr
, paddr
+ size
);
90 dmac_unmap_area(__va(paddr
), size
, dir
);
94 static dma_addr_t
arm_nommu_dma_map_page(struct device
*dev
, struct page
*page
,
95 unsigned long offset
, size_t size
,
96 enum dma_data_direction dir
,
99 dma_addr_t handle
= page_to_phys(page
) + offset
;
101 __dma_page_cpu_to_dev(handle
, size
, dir
);
106 static void arm_nommu_dma_unmap_page(struct device
*dev
, dma_addr_t handle
,
107 size_t size
, enum dma_data_direction dir
,
110 __dma_page_dev_to_cpu(handle
, size
, dir
);
114 static int arm_nommu_dma_map_sg(struct device
*dev
, struct scatterlist
*sgl
,
115 int nents
, enum dma_data_direction dir
,
119 struct scatterlist
*sg
;
121 for_each_sg(sgl
, sg
, nents
, i
) {
122 sg_dma_address(sg
) = sg_phys(sg
);
123 sg_dma_len(sg
) = sg
->length
;
124 __dma_page_cpu_to_dev(sg_dma_address(sg
), sg_dma_len(sg
), dir
);
130 static void arm_nommu_dma_unmap_sg(struct device
*dev
, struct scatterlist
*sgl
,
131 int nents
, enum dma_data_direction dir
,
134 struct scatterlist
*sg
;
137 for_each_sg(sgl
, sg
, nents
, i
)
138 __dma_page_dev_to_cpu(sg_dma_address(sg
), sg_dma_len(sg
), dir
);
141 static void arm_nommu_dma_sync_single_for_device(struct device
*dev
,
142 dma_addr_t handle
, size_t size
, enum dma_data_direction dir
)
144 __dma_page_cpu_to_dev(handle
, size
, dir
);
147 static void arm_nommu_dma_sync_single_for_cpu(struct device
*dev
,
148 dma_addr_t handle
, size_t size
, enum dma_data_direction dir
)
150 __dma_page_cpu_to_dev(handle
, size
, dir
);
153 static void arm_nommu_dma_sync_sg_for_device(struct device
*dev
, struct scatterlist
*sgl
,
154 int nents
, enum dma_data_direction dir
)
156 struct scatterlist
*sg
;
159 for_each_sg(sgl
, sg
, nents
, i
)
160 __dma_page_cpu_to_dev(sg_dma_address(sg
), sg_dma_len(sg
), dir
);
163 static void arm_nommu_dma_sync_sg_for_cpu(struct device
*dev
, struct scatterlist
*sgl
,
164 int nents
, enum dma_data_direction dir
)
166 struct scatterlist
*sg
;
169 for_each_sg(sgl
, sg
, nents
, i
)
170 __dma_page_dev_to_cpu(sg_dma_address(sg
), sg_dma_len(sg
), dir
);
173 const struct dma_map_ops arm_nommu_dma_ops
= {
174 .alloc
= arm_nommu_dma_alloc
,
175 .free
= arm_nommu_dma_free
,
176 .map_page
= arm_nommu_dma_map_page
,
177 .unmap_page
= arm_nommu_dma_unmap_page
,
178 .map_sg
= arm_nommu_dma_map_sg
,
179 .unmap_sg
= arm_nommu_dma_unmap_sg
,
180 .sync_single_for_device
= arm_nommu_dma_sync_single_for_device
,
181 .sync_single_for_cpu
= arm_nommu_dma_sync_single_for_cpu
,
182 .sync_sg_for_device
= arm_nommu_dma_sync_sg_for_device
,
183 .sync_sg_for_cpu
= arm_nommu_dma_sync_sg_for_cpu
,
185 EXPORT_SYMBOL(arm_nommu_dma_ops
);
187 static const struct dma_map_ops
*arm_nommu_get_dma_map_ops(bool coherent
)
189 return coherent
? &dma_noop_ops
: &arm_nommu_dma_ops
;
192 void arch_setup_dma_ops(struct device
*dev
, u64 dma_base
, u64 size
,
193 const struct iommu_ops
*iommu
, bool coherent
)
195 const struct dma_map_ops
*dma_ops
;
197 if (IS_ENABLED(CONFIG_CPU_V7M
)) {
199 * Cache support for v7m is optional, so can be treated as
200 * coherent if no cache has been detected. Note that it is not
201 * enough to check if MPU is in use or not since in absense of
202 * MPU system memory map is used.
204 dev
->archdata
.dma_coherent
= (cacheid
) ? coherent
: true;
207 * Assume coherent DMA in case MMU/MPU has not been set up.
209 dev
->archdata
.dma_coherent
= (get_cr() & CR_M
) ? coherent
: true;
212 dma_ops
= arm_nommu_get_dma_map_ops(dev
->archdata
.dma_coherent
);
214 set_dma_ops(dev
, dma_ops
);
217 void arch_teardown_dma_ops(struct device
*dev
)
221 #define PREALLOC_DMA_DEBUG_ENTRIES 4096
223 static int __init
dma_debug_do_init(void)
225 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
228 core_initcall(dma_debug_do_init
);