]>
Commit | Line | Data |
---|---|---|
ee7e5516 DES |
1 | /* |
2 | * Coherent per-device memory handling. | |
3 | * Borrowed from i386 | |
4 | */ | |
6b03ae0d | 5 | #include <linux/io.h> |
5a0e3ad6 | 6 | #include <linux/slab.h> |
ee7e5516 | 7 | #include <linux/kernel.h> |
08a999ce | 8 | #include <linux/module.h> |
ee7e5516 DES |
9 | #include <linux/dma-mapping.h> |
10 | ||
11 | struct dma_coherent_mem { | |
12 | void *virt_base; | |
ed1d218c | 13 | dma_addr_t device_base; |
88a984ba | 14 | unsigned long pfn_base; |
ee7e5516 DES |
15 | int size; |
16 | int flags; | |
17 | unsigned long *bitmap; | |
7bfa5ab6 | 18 | spinlock_t spinlock; |
c41f9ea9 | 19 | bool use_dev_dma_pfn_offset; |
ee7e5516 DES |
20 | }; |
21 | ||
93228b44 VM |
22 | static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init; |
23 | ||
24 | static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *dev) | |
25 | { | |
26 | if (dev && dev->dma_mem) | |
27 | return dev->dma_mem; | |
28 | return dma_coherent_default_memory; | |
29 | } | |
30 | ||
c41f9ea9 VM |
31 | static inline dma_addr_t dma_get_device_base(struct device *dev, |
32 | struct dma_coherent_mem * mem) | |
33 | { | |
34 | if (mem->use_dev_dma_pfn_offset) | |
35 | return (mem->pfn_base - dev->dma_pfn_offset) << PAGE_SHIFT; | |
36 | else | |
37 | return mem->device_base; | |
38 | } | |
39 | ||
9e5b3d6f MN |
40 | static bool dma_init_coherent_memory( |
41 | phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags, | |
42 | struct dma_coherent_mem **mem) | |
ee7e5516 | 43 | { |
7bfa5ab6 | 44 | struct dma_coherent_mem *dma_mem = NULL; |
ee7e5516 DES |
45 | void __iomem *mem_base = NULL; |
46 | int pages = size >> PAGE_SHIFT; | |
47 | int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); | |
48 | ||
49 | if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0) | |
50 | goto out; | |
51 | if (!size) | |
52 | goto out; | |
ee7e5516 | 53 | |
6b03ae0d BS |
54 | if (flags & DMA_MEMORY_MAP) |
55 | mem_base = memremap(phys_addr, size, MEMREMAP_WC); | |
56 | else | |
57 | mem_base = ioremap(phys_addr, size); | |
ee7e5516 DES |
58 | if (!mem_base) |
59 | goto out; | |
60 | ||
7bfa5ab6 MS |
61 | dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); |
62 | if (!dma_mem) | |
ee7e5516 | 63 | goto out; |
7bfa5ab6 MS |
64 | dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); |
65 | if (!dma_mem->bitmap) | |
66 | goto out; | |
67 | ||
68 | dma_mem->virt_base = mem_base; | |
69 | dma_mem->device_base = device_addr; | |
70 | dma_mem->pfn_base = PFN_DOWN(phys_addr); | |
71 | dma_mem->size = pages; | |
72 | dma_mem->flags = flags; | |
73 | spin_lock_init(&dma_mem->spinlock); | |
ee7e5516 | 74 | |
7bfa5ab6 | 75 | *mem = dma_mem; |
9e5b3d6f | 76 | return true; |
ee7e5516 | 77 | |
7bfa5ab6 MS |
78 | out: |
79 | kfree(dma_mem); | |
6b03ae0d BS |
80 | if (mem_base) { |
81 | if (flags & DMA_MEMORY_MAP) | |
82 | memunmap(mem_base); | |
83 | else | |
84 | iounmap(mem_base); | |
85 | } | |
9e5b3d6f | 86 | return false; |
ee7e5516 | 87 | } |
7bfa5ab6 MS |
88 | |
89 | static void dma_release_coherent_memory(struct dma_coherent_mem *mem) | |
90 | { | |
91 | if (!mem) | |
92 | return; | |
6b03ae0d BS |
93 | |
94 | if (mem->flags & DMA_MEMORY_MAP) | |
95 | memunmap(mem->virt_base); | |
96 | else | |
97 | iounmap(mem->virt_base); | |
7bfa5ab6 MS |
98 | kfree(mem->bitmap); |
99 | kfree(mem); | |
100 | } | |
101 | ||
102 | static int dma_assign_coherent_memory(struct device *dev, | |
103 | struct dma_coherent_mem *mem) | |
104 | { | |
93228b44 VM |
105 | if (!dev) |
106 | return -ENODEV; | |
107 | ||
7bfa5ab6 MS |
108 | if (dev->dma_mem) |
109 | return -EBUSY; | |
110 | ||
111 | dev->dma_mem = mem; | |
112 | /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */ | |
113 | ||
114 | return 0; | |
115 | } | |
116 | ||
117 | int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, | |
118 | dma_addr_t device_addr, size_t size, int flags) | |
119 | { | |
120 | struct dma_coherent_mem *mem; | |
7bfa5ab6 | 121 | |
9e5b3d6f MN |
122 | if (!dma_init_coherent_memory(phys_addr, device_addr, size, flags, |
123 | &mem)) | |
7bfa5ab6 MS |
124 | return 0; |
125 | ||
126 | if (dma_assign_coherent_memory(dev, mem) == 0) | |
9e5b3d6f | 127 | return flags & DMA_MEMORY_MAP ? DMA_MEMORY_MAP : DMA_MEMORY_IO; |
7bfa5ab6 MS |
128 | |
129 | dma_release_coherent_memory(mem); | |
130 | return 0; | |
131 | } | |
ee7e5516 DES |
132 | EXPORT_SYMBOL(dma_declare_coherent_memory); |
133 | ||
134 | void dma_release_declared_memory(struct device *dev) | |
135 | { | |
136 | struct dma_coherent_mem *mem = dev->dma_mem; | |
137 | ||
138 | if (!mem) | |
139 | return; | |
7bfa5ab6 | 140 | dma_release_coherent_memory(mem); |
ee7e5516 | 141 | dev->dma_mem = NULL; |
ee7e5516 DES |
142 | } |
143 | EXPORT_SYMBOL(dma_release_declared_memory); | |
144 | ||
145 | void *dma_mark_declared_memory_occupied(struct device *dev, | |
146 | dma_addr_t device_addr, size_t size) | |
147 | { | |
148 | struct dma_coherent_mem *mem = dev->dma_mem; | |
7bfa5ab6 | 149 | unsigned long flags; |
ee7e5516 | 150 | int pos, err; |
ee7e5516 | 151 | |
d2dc1f4a | 152 | size += device_addr & ~PAGE_MASK; |
ee7e5516 DES |
153 | |
154 | if (!mem) | |
155 | return ERR_PTR(-EINVAL); | |
156 | ||
7bfa5ab6 | 157 | spin_lock_irqsave(&mem->spinlock, flags); |
c41f9ea9 | 158 | pos = PFN_DOWN(device_addr - dma_get_device_base(dev, mem)); |
d2dc1f4a | 159 | err = bitmap_allocate_region(mem->bitmap, pos, get_order(size)); |
7bfa5ab6 MS |
160 | spin_unlock_irqrestore(&mem->spinlock, flags); |
161 | ||
ee7e5516 DES |
162 | if (err != 0) |
163 | return ERR_PTR(err); | |
164 | return mem->virt_base + (pos << PAGE_SHIFT); | |
165 | } | |
166 | EXPORT_SYMBOL(dma_mark_declared_memory_occupied); | |
167 | ||
b6d4f7e3 | 168 | /** |
cb3952bf | 169 | * dma_alloc_from_coherent() - try to allocate memory from the per-device coherent area |
b6d4f7e3 DES |
170 | * |
171 | * @dev: device from which we allocate memory | |
172 | * @size: size of requested memory area | |
173 | * @dma_handle: This will be filled with the correct dma handle | |
174 | * @ret: This pointer will be filled with the virtual address | |
0609697e | 175 | * to allocated area. |
b6d4f7e3 | 176 | * |
cb3952bf | 177 | * This function should be only called from per-arch dma_alloc_coherent() |
b6d4f7e3 DES |
178 | * to support allocation from per-device coherent memory pools. |
179 | * | |
180 | * Returns 0 if dma_alloc_coherent should continue with allocating from | |
cb3952bf | 181 | * generic memory areas, or !0 if dma_alloc_coherent should return @ret. |
b6d4f7e3 | 182 | */ |
ee7e5516 DES |
183 | int dma_alloc_from_coherent(struct device *dev, ssize_t size, |
184 | dma_addr_t *dma_handle, void **ret) | |
185 | { | |
93228b44 | 186 | struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); |
ee7e5516 | 187 | int order = get_order(size); |
7bfa5ab6 | 188 | unsigned long flags; |
eccd83e1 | 189 | int pageno; |
dd01c75f | 190 | int dma_memory_map; |
ee7e5516 | 191 | |
eccd83e1 AM |
192 | if (!mem) |
193 | return 0; | |
0609697e PM |
194 | |
195 | *ret = NULL; | |
7bfa5ab6 | 196 | spin_lock_irqsave(&mem->spinlock, flags); |
0609697e | 197 | |
cdf57cab | 198 | if (unlikely(size > (mem->size << PAGE_SHIFT))) |
0609697e | 199 | goto err; |
eccd83e1 AM |
200 | |
201 | pageno = bitmap_find_free_region(mem->bitmap, mem->size, order); | |
0609697e PM |
202 | if (unlikely(pageno < 0)) |
203 | goto err; | |
204 | ||
205 | /* | |
206 | * Memory was found in the per-device area. | |
207 | */ | |
c41f9ea9 | 208 | *dma_handle = dma_get_device_base(dev, mem) + (pageno << PAGE_SHIFT); |
0609697e | 209 | *ret = mem->virt_base + (pageno << PAGE_SHIFT); |
dd01c75f BH |
210 | dma_memory_map = (mem->flags & DMA_MEMORY_MAP); |
211 | spin_unlock_irqrestore(&mem->spinlock, flags); | |
212 | if (dma_memory_map) | |
20d7a35b BS |
213 | memset(*ret, 0, size); |
214 | else | |
215 | memset_io(*ret, 0, size); | |
0609697e | 216 | |
eccd83e1 | 217 | return 1; |
0609697e PM |
218 | |
219 | err: | |
7bfa5ab6 | 220 | spin_unlock_irqrestore(&mem->spinlock, flags); |
0609697e PM |
221 | /* |
222 | * In the case where the allocation can not be satisfied from the | |
223 | * per-device area, try to fall back to generic memory if the | |
224 | * constraints allow it. | |
225 | */ | |
226 | return mem->flags & DMA_MEMORY_EXCLUSIVE; | |
ee7e5516 | 227 | } |
a38409fb | 228 | EXPORT_SYMBOL(dma_alloc_from_coherent); |
ee7e5516 | 229 | |
b6d4f7e3 | 230 | /** |
cb3952bf | 231 | * dma_release_from_coherent() - try to free the memory allocated from per-device coherent memory pool |
b6d4f7e3 DES |
232 | * @dev: device from which the memory was allocated |
233 | * @order: the order of pages allocated | |
234 | * @vaddr: virtual address of allocated pages | |
235 | * | |
236 | * This checks whether the memory was allocated from the per-device | |
237 | * coherent memory pool and if so, releases that memory. | |
238 | * | |
239 | * Returns 1 if we correctly released the memory, or 0 if | |
cb3952bf | 240 | * dma_release_coherent() should proceed with releasing memory from |
b6d4f7e3 DES |
241 | * generic pools. |
242 | */ | |
ee7e5516 DES |
243 | int dma_release_from_coherent(struct device *dev, int order, void *vaddr) |
244 | { | |
93228b44 | 245 | struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); |
ee7e5516 DES |
246 | |
247 | if (mem && vaddr >= mem->virt_base && vaddr < | |
248 | (mem->virt_base + (mem->size << PAGE_SHIFT))) { | |
249 | int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; | |
7bfa5ab6 | 250 | unsigned long flags; |
ee7e5516 | 251 | |
7bfa5ab6 | 252 | spin_lock_irqsave(&mem->spinlock, flags); |
ee7e5516 | 253 | bitmap_release_region(mem->bitmap, page, order); |
7bfa5ab6 | 254 | spin_unlock_irqrestore(&mem->spinlock, flags); |
ee7e5516 DES |
255 | return 1; |
256 | } | |
257 | return 0; | |
258 | } | |
a38409fb | 259 | EXPORT_SYMBOL(dma_release_from_coherent); |
bca0fa5f MS |
260 | |
261 | /** | |
262 | * dma_mmap_from_coherent() - try to mmap the memory allocated from | |
263 | * per-device coherent memory pool to userspace | |
264 | * @dev: device from which the memory was allocated | |
265 | * @vma: vm_area for the userspace memory | |
266 | * @vaddr: cpu address returned by dma_alloc_from_coherent | |
267 | * @size: size of the memory buffer allocated by dma_alloc_from_coherent | |
6e7b4a59 | 268 | * @ret: result from remap_pfn_range() |
bca0fa5f MS |
269 | * |
270 | * This checks whether the memory was allocated from the per-device | |
271 | * coherent memory pool and if so, maps that memory to the provided vma. | |
272 | * | |
ba4d93bc LP |
273 | * Returns 1 if we correctly mapped the memory, or 0 if the caller should |
274 | * proceed with mapping memory from generic pools. | |
bca0fa5f MS |
275 | */ |
276 | int dma_mmap_from_coherent(struct device *dev, struct vm_area_struct *vma, | |
277 | void *vaddr, size_t size, int *ret) | |
278 | { | |
93228b44 | 279 | struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); |
bca0fa5f MS |
280 | |
281 | if (mem && vaddr >= mem->virt_base && vaddr + size <= | |
282 | (mem->virt_base + (mem->size << PAGE_SHIFT))) { | |
283 | unsigned long off = vma->vm_pgoff; | |
284 | int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; | |
e688f144 | 285 | int user_count = vma_pages(vma); |
9ca5d4fd | 286 | int count = PAGE_ALIGN(size) >> PAGE_SHIFT; |
bca0fa5f MS |
287 | |
288 | *ret = -ENXIO; | |
289 | if (off < count && user_count <= count - off) { | |
88a984ba | 290 | unsigned long pfn = mem->pfn_base + start + off; |
bca0fa5f MS |
291 | *ret = remap_pfn_range(vma, vma->vm_start, pfn, |
292 | user_count << PAGE_SHIFT, | |
293 | vma->vm_page_prot); | |
294 | } | |
295 | return 1; | |
296 | } | |
297 | return 0; | |
298 | } | |
299 | EXPORT_SYMBOL(dma_mmap_from_coherent); | |
7bfa5ab6 MS |
300 | |
301 | /* | |
302 | * Support for reserved memory regions defined in device tree | |
303 | */ | |
304 | #ifdef CONFIG_OF_RESERVED_MEM | |
305 | #include <linux/of.h> | |
306 | #include <linux/of_fdt.h> | |
307 | #include <linux/of_reserved_mem.h> | |
308 | ||
93228b44 VM |
309 | static struct reserved_mem *dma_reserved_default_memory __initdata; |
310 | ||
7bfa5ab6 MS |
311 | static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev) |
312 | { | |
313 | struct dma_coherent_mem *mem = rmem->priv; | |
314 | ||
315 | if (!mem && | |
9e5b3d6f MN |
316 | !dma_init_coherent_memory(rmem->base, rmem->base, rmem->size, |
317 | DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE, | |
318 | &mem)) { | |
7bfa5ab6 MS |
319 | pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n", |
320 | &rmem->base, (unsigned long)rmem->size / SZ_1M); | |
321 | return -ENODEV; | |
322 | } | |
c41f9ea9 | 323 | mem->use_dev_dma_pfn_offset = true; |
7bfa5ab6 MS |
324 | rmem->priv = mem; |
325 | dma_assign_coherent_memory(dev, mem); | |
326 | return 0; | |
327 | } | |
328 | ||
329 | static void rmem_dma_device_release(struct reserved_mem *rmem, | |
330 | struct device *dev) | |
331 | { | |
93228b44 VM |
332 | if (dev) |
333 | dev->dma_mem = NULL; | |
7bfa5ab6 MS |
334 | } |
335 | ||
336 | static const struct reserved_mem_ops rmem_dma_ops = { | |
337 | .device_init = rmem_dma_device_init, | |
338 | .device_release = rmem_dma_device_release, | |
339 | }; | |
340 | ||
341 | static int __init rmem_dma_setup(struct reserved_mem *rmem) | |
342 | { | |
343 | unsigned long node = rmem->fdt_node; | |
344 | ||
345 | if (of_get_flat_dt_prop(node, "reusable", NULL)) | |
346 | return -EINVAL; | |
347 | ||
348 | #ifdef CONFIG_ARM | |
349 | if (!of_get_flat_dt_prop(node, "no-map", NULL)) { | |
350 | pr_err("Reserved memory: regions without no-map are not yet supported\n"); | |
351 | return -EINVAL; | |
352 | } | |
93228b44 VM |
353 | |
354 | if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) { | |
355 | WARN(dma_reserved_default_memory, | |
356 | "Reserved memory: region for default DMA coherent area is redefined\n"); | |
357 | dma_reserved_default_memory = rmem; | |
358 | } | |
7bfa5ab6 MS |
359 | #endif |
360 | ||
361 | rmem->ops = &rmem_dma_ops; | |
362 | pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n", | |
363 | &rmem->base, (unsigned long)rmem->size / SZ_1M); | |
364 | return 0; | |
365 | } | |
93228b44 VM |
366 | |
367 | static int __init dma_init_reserved_memory(void) | |
368 | { | |
369 | const struct reserved_mem_ops *ops; | |
370 | int ret; | |
371 | ||
372 | if (!dma_reserved_default_memory) | |
373 | return -ENOMEM; | |
374 | ||
375 | ops = dma_reserved_default_memory->ops; | |
376 | ||
377 | /* | |
378 | * We rely on rmem_dma_device_init() does not propagate error of | |
379 | * dma_assign_coherent_memory() for "NULL" device. | |
380 | */ | |
381 | ret = ops->device_init(dma_reserved_default_memory, NULL); | |
382 | ||
383 | if (!ret) { | |
384 | dma_coherent_default_memory = dma_reserved_default_memory->priv; | |
385 | pr_info("DMA: default coherent area is set\n"); | |
386 | } | |
387 | ||
388 | return ret; | |
389 | } | |
390 | ||
391 | core_initcall(dma_init_reserved_memory); | |
392 | ||
7bfa5ab6 MS |
393 | RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup); |
394 | #endif |