]>
Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
ee7e5516 DES |
2 | /* |
3 | * Coherent per-device memory handling. | |
4 | * Borrowed from i386 | |
5 | */ | |
6b03ae0d | 6 | #include <linux/io.h> |
5a0e3ad6 | 7 | #include <linux/slab.h> |
ee7e5516 | 8 | #include <linux/kernel.h> |
08a999ce | 9 | #include <linux/module.h> |
ee7e5516 DES |
10 | #include <linux/dma-mapping.h> |
11 | ||
12 | struct dma_coherent_mem { | |
13 | void *virt_base; | |
ed1d218c | 14 | dma_addr_t device_base; |
88a984ba | 15 | unsigned long pfn_base; |
ee7e5516 DES |
16 | int size; |
17 | int flags; | |
18 | unsigned long *bitmap; | |
7bfa5ab6 | 19 | spinlock_t spinlock; |
c41f9ea9 | 20 | bool use_dev_dma_pfn_offset; |
ee7e5516 DES |
21 | }; |
22 | ||
93228b44 VM |
23 | static struct dma_coherent_mem *dma_coherent_default_memory __ro_after_init; |
24 | ||
25 | static inline struct dma_coherent_mem *dev_get_coherent_memory(struct device *dev) | |
26 | { | |
27 | if (dev && dev->dma_mem) | |
28 | return dev->dma_mem; | |
43fc509c | 29 | return NULL; |
93228b44 VM |
30 | } |
31 | ||
c41f9ea9 VM |
32 | static inline dma_addr_t dma_get_device_base(struct device *dev, |
33 | struct dma_coherent_mem * mem) | |
34 | { | |
35 | if (mem->use_dev_dma_pfn_offset) | |
36 | return (mem->pfn_base - dev->dma_pfn_offset) << PAGE_SHIFT; | |
37 | else | |
38 | return mem->device_base; | |
39 | } | |
40 | ||
d35b0996 | 41 | static int dma_init_coherent_memory( |
9e5b3d6f MN |
42 | phys_addr_t phys_addr, dma_addr_t device_addr, size_t size, int flags, |
43 | struct dma_coherent_mem **mem) | |
ee7e5516 | 44 | { |
7bfa5ab6 | 45 | struct dma_coherent_mem *dma_mem = NULL; |
ee7e5516 DES |
46 | void __iomem *mem_base = NULL; |
47 | int pages = size >> PAGE_SHIFT; | |
48 | int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long); | |
d35b0996 | 49 | int ret; |
ee7e5516 | 50 | |
d35b0996 AB |
51 | if (!size) { |
52 | ret = -EINVAL; | |
ee7e5516 | 53 | goto out; |
d35b0996 | 54 | } |
ee7e5516 | 55 | |
2436bdcd | 56 | mem_base = memremap(phys_addr, size, MEMREMAP_WC); |
d35b0996 AB |
57 | if (!mem_base) { |
58 | ret = -EINVAL; | |
ee7e5516 | 59 | goto out; |
d35b0996 | 60 | } |
7bfa5ab6 | 61 | dma_mem = kzalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL); |
d35b0996 AB |
62 | if (!dma_mem) { |
63 | ret = -ENOMEM; | |
ee7e5516 | 64 | goto out; |
d35b0996 | 65 | } |
7bfa5ab6 | 66 | dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); |
d35b0996 AB |
67 | if (!dma_mem->bitmap) { |
68 | ret = -ENOMEM; | |
7bfa5ab6 | 69 | goto out; |
d35b0996 | 70 | } |
7bfa5ab6 MS |
71 | |
72 | dma_mem->virt_base = mem_base; | |
73 | dma_mem->device_base = device_addr; | |
74 | dma_mem->pfn_base = PFN_DOWN(phys_addr); | |
75 | dma_mem->size = pages; | |
76 | dma_mem->flags = flags; | |
77 | spin_lock_init(&dma_mem->spinlock); | |
ee7e5516 | 78 | |
7bfa5ab6 | 79 | *mem = dma_mem; |
d35b0996 | 80 | return 0; |
ee7e5516 | 81 | |
7bfa5ab6 MS |
82 | out: |
83 | kfree(dma_mem); | |
2436bdcd CH |
84 | if (mem_base) |
85 | memunmap(mem_base); | |
d35b0996 | 86 | return ret; |
ee7e5516 | 87 | } |
7bfa5ab6 MS |
88 | |
89 | static void dma_release_coherent_memory(struct dma_coherent_mem *mem) | |
90 | { | |
91 | if (!mem) | |
92 | return; | |
6b03ae0d | 93 | |
2436bdcd | 94 | memunmap(mem->virt_base); |
7bfa5ab6 MS |
95 | kfree(mem->bitmap); |
96 | kfree(mem); | |
97 | } | |
98 | ||
99 | static int dma_assign_coherent_memory(struct device *dev, | |
100 | struct dma_coherent_mem *mem) | |
101 | { | |
93228b44 VM |
102 | if (!dev) |
103 | return -ENODEV; | |
104 | ||
7bfa5ab6 MS |
105 | if (dev->dma_mem) |
106 | return -EBUSY; | |
107 | ||
108 | dev->dma_mem = mem; | |
7bfa5ab6 MS |
109 | return 0; |
110 | } | |
111 | ||
112 | int dma_declare_coherent_memory(struct device *dev, phys_addr_t phys_addr, | |
113 | dma_addr_t device_addr, size_t size, int flags) | |
114 | { | |
115 | struct dma_coherent_mem *mem; | |
2436bdcd | 116 | int ret; |
7bfa5ab6 | 117 | |
2436bdcd CH |
118 | ret = dma_init_coherent_memory(phys_addr, device_addr, size, flags, &mem); |
119 | if (ret) | |
120 | return ret; | |
7bfa5ab6 | 121 | |
2436bdcd CH |
122 | ret = dma_assign_coherent_memory(dev, mem); |
123 | if (ret) | |
124 | dma_release_coherent_memory(mem); | |
125 | return ret; | |
7bfa5ab6 | 126 | } |
ee7e5516 DES |
127 | EXPORT_SYMBOL(dma_declare_coherent_memory); |
128 | ||
129 | void dma_release_declared_memory(struct device *dev) | |
130 | { | |
131 | struct dma_coherent_mem *mem = dev->dma_mem; | |
132 | ||
133 | if (!mem) | |
134 | return; | |
7bfa5ab6 | 135 | dma_release_coherent_memory(mem); |
ee7e5516 | 136 | dev->dma_mem = NULL; |
ee7e5516 DES |
137 | } |
138 | EXPORT_SYMBOL(dma_release_declared_memory); | |
139 | ||
140 | void *dma_mark_declared_memory_occupied(struct device *dev, | |
141 | dma_addr_t device_addr, size_t size) | |
142 | { | |
143 | struct dma_coherent_mem *mem = dev->dma_mem; | |
7bfa5ab6 | 144 | unsigned long flags; |
ee7e5516 | 145 | int pos, err; |
ee7e5516 | 146 | |
d2dc1f4a | 147 | size += device_addr & ~PAGE_MASK; |
ee7e5516 DES |
148 | |
149 | if (!mem) | |
150 | return ERR_PTR(-EINVAL); | |
151 | ||
7bfa5ab6 | 152 | spin_lock_irqsave(&mem->spinlock, flags); |
c41f9ea9 | 153 | pos = PFN_DOWN(device_addr - dma_get_device_base(dev, mem)); |
d2dc1f4a | 154 | err = bitmap_allocate_region(mem->bitmap, pos, get_order(size)); |
7bfa5ab6 MS |
155 | spin_unlock_irqrestore(&mem->spinlock, flags); |
156 | ||
ee7e5516 DES |
157 | if (err != 0) |
158 | return ERR_PTR(err); | |
159 | return mem->virt_base + (pos << PAGE_SHIFT); | |
160 | } | |
161 | EXPORT_SYMBOL(dma_mark_declared_memory_occupied); | |
162 | ||
43fc509c VM |
163 | static void *__dma_alloc_from_coherent(struct dma_coherent_mem *mem, |
164 | ssize_t size, dma_addr_t *dma_handle) | |
ee7e5516 | 165 | { |
ee7e5516 | 166 | int order = get_order(size); |
7bfa5ab6 | 167 | unsigned long flags; |
eccd83e1 | 168 | int pageno; |
43fc509c | 169 | void *ret; |
ee7e5516 | 170 | |
7bfa5ab6 | 171 | spin_lock_irqsave(&mem->spinlock, flags); |
0609697e | 172 | |
cdf57cab | 173 | if (unlikely(size > (mem->size << PAGE_SHIFT))) |
0609697e | 174 | goto err; |
eccd83e1 AM |
175 | |
176 | pageno = bitmap_find_free_region(mem->bitmap, mem->size, order); | |
0609697e PM |
177 | if (unlikely(pageno < 0)) |
178 | goto err; | |
179 | ||
180 | /* | |
43fc509c | 181 | * Memory was found in the coherent area. |
0609697e | 182 | */ |
43fc509c VM |
183 | *dma_handle = mem->device_base + (pageno << PAGE_SHIFT); |
184 | ret = mem->virt_base + (pageno << PAGE_SHIFT); | |
dd01c75f | 185 | spin_unlock_irqrestore(&mem->spinlock, flags); |
2436bdcd | 186 | memset(ret, 0, size); |
43fc509c | 187 | return ret; |
0609697e | 188 | err: |
7bfa5ab6 | 189 | spin_unlock_irqrestore(&mem->spinlock, flags); |
43fc509c VM |
190 | return NULL; |
191 | } | |
192 | ||
193 | /** | |
194 | * dma_alloc_from_dev_coherent() - allocate memory from device coherent pool | |
195 | * @dev: device from which we allocate memory | |
196 | * @size: size of requested memory area | |
197 | * @dma_handle: This will be filled with the correct dma handle | |
198 | * @ret: This pointer will be filled with the virtual address | |
199 | * to allocated area. | |
200 | * | |
201 | * This function should be only called from per-arch dma_alloc_coherent() | |
202 | * to support allocation from per-device coherent memory pools. | |
203 | * | |
204 | * Returns 0 if dma_alloc_coherent should continue with allocating from | |
205 | * generic memory areas, or !0 if dma_alloc_coherent should return @ret. | |
206 | */ | |
207 | int dma_alloc_from_dev_coherent(struct device *dev, ssize_t size, | |
208 | dma_addr_t *dma_handle, void **ret) | |
209 | { | |
210 | struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); | |
211 | ||
212 | if (!mem) | |
213 | return 0; | |
214 | ||
215 | *ret = __dma_alloc_from_coherent(mem, size, dma_handle); | |
216 | if (*ret) | |
217 | return 1; | |
218 | ||
0609697e PM |
219 | /* |
220 | * In the case where the allocation can not be satisfied from the | |
221 | * per-device area, try to fall back to generic memory if the | |
222 | * constraints allow it. | |
223 | */ | |
224 | return mem->flags & DMA_MEMORY_EXCLUSIVE; | |
ee7e5516 | 225 | } |
43fc509c | 226 | EXPORT_SYMBOL(dma_alloc_from_dev_coherent); |
ee7e5516 | 227 | |
43fc509c | 228 | void *dma_alloc_from_global_coherent(ssize_t size, dma_addr_t *dma_handle) |
ee7e5516 | 229 | { |
43fc509c VM |
230 | if (!dma_coherent_default_memory) |
231 | return NULL; | |
232 | ||
233 | return __dma_alloc_from_coherent(dma_coherent_default_memory, size, | |
234 | dma_handle); | |
235 | } | |
ee7e5516 | 236 | |
43fc509c VM |
237 | static int __dma_release_from_coherent(struct dma_coherent_mem *mem, |
238 | int order, void *vaddr) | |
239 | { | |
ee7e5516 DES |
240 | if (mem && vaddr >= mem->virt_base && vaddr < |
241 | (mem->virt_base + (mem->size << PAGE_SHIFT))) { | |
242 | int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; | |
7bfa5ab6 | 243 | unsigned long flags; |
ee7e5516 | 244 | |
7bfa5ab6 | 245 | spin_lock_irqsave(&mem->spinlock, flags); |
ee7e5516 | 246 | bitmap_release_region(mem->bitmap, page, order); |
7bfa5ab6 | 247 | spin_unlock_irqrestore(&mem->spinlock, flags); |
ee7e5516 DES |
248 | return 1; |
249 | } | |
250 | return 0; | |
251 | } | |
bca0fa5f MS |
252 | |
253 | /** | |
43fc509c | 254 | * dma_release_from_dev_coherent() - free memory to device coherent memory pool |
bca0fa5f | 255 | * @dev: device from which the memory was allocated |
43fc509c VM |
256 | * @order: the order of pages allocated |
257 | * @vaddr: virtual address of allocated pages | |
bca0fa5f MS |
258 | * |
259 | * This checks whether the memory was allocated from the per-device | |
43fc509c | 260 | * coherent memory pool and if so, releases that memory. |
bca0fa5f | 261 | * |
43fc509c VM |
262 | * Returns 1 if we correctly released the memory, or 0 if the caller should |
263 | * proceed with releasing memory from generic pools. | |
bca0fa5f | 264 | */ |
43fc509c | 265 | int dma_release_from_dev_coherent(struct device *dev, int order, void *vaddr) |
bca0fa5f | 266 | { |
93228b44 | 267 | struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); |
bca0fa5f | 268 | |
43fc509c VM |
269 | return __dma_release_from_coherent(mem, order, vaddr); |
270 | } | |
271 | EXPORT_SYMBOL(dma_release_from_dev_coherent); | |
272 | ||
273 | int dma_release_from_global_coherent(int order, void *vaddr) | |
274 | { | |
275 | if (!dma_coherent_default_memory) | |
276 | return 0; | |
277 | ||
278 | return __dma_release_from_coherent(dma_coherent_default_memory, order, | |
279 | vaddr); | |
280 | } | |
281 | ||
282 | static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem, | |
283 | struct vm_area_struct *vma, void *vaddr, size_t size, int *ret) | |
284 | { | |
bca0fa5f MS |
285 | if (mem && vaddr >= mem->virt_base && vaddr + size <= |
286 | (mem->virt_base + (mem->size << PAGE_SHIFT))) { | |
287 | unsigned long off = vma->vm_pgoff; | |
288 | int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; | |
e688f144 | 289 | int user_count = vma_pages(vma); |
9ca5d4fd | 290 | int count = PAGE_ALIGN(size) >> PAGE_SHIFT; |
bca0fa5f MS |
291 | |
292 | *ret = -ENXIO; | |
293 | if (off < count && user_count <= count - off) { | |
88a984ba | 294 | unsigned long pfn = mem->pfn_base + start + off; |
bca0fa5f MS |
295 | *ret = remap_pfn_range(vma, vma->vm_start, pfn, |
296 | user_count << PAGE_SHIFT, | |
297 | vma->vm_page_prot); | |
298 | } | |
299 | return 1; | |
300 | } | |
301 | return 0; | |
302 | } | |
43fc509c VM |
303 | |
304 | /** | |
305 | * dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool | |
306 | * @dev: device from which the memory was allocated | |
307 | * @vma: vm_area for the userspace memory | |
308 | * @vaddr: cpu address returned by dma_alloc_from_dev_coherent | |
309 | * @size: size of the memory buffer allocated | |
310 | * @ret: result from remap_pfn_range() | |
311 | * | |
312 | * This checks whether the memory was allocated from the per-device | |
313 | * coherent memory pool and if so, maps that memory to the provided vma. | |
314 | * | |
315 | * Returns 1 if we correctly mapped the memory, or 0 if the caller should | |
316 | * proceed with mapping memory from generic pools. | |
317 | */ | |
318 | int dma_mmap_from_dev_coherent(struct device *dev, struct vm_area_struct *vma, | |
319 | void *vaddr, size_t size, int *ret) | |
320 | { | |
321 | struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); | |
322 | ||
323 | return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret); | |
324 | } | |
325 | EXPORT_SYMBOL(dma_mmap_from_dev_coherent); | |
326 | ||
327 | int dma_mmap_from_global_coherent(struct vm_area_struct *vma, void *vaddr, | |
328 | size_t size, int *ret) | |
329 | { | |
330 | if (!dma_coherent_default_memory) | |
331 | return 0; | |
332 | ||
333 | return __dma_mmap_from_coherent(dma_coherent_default_memory, vma, | |
334 | vaddr, size, ret); | |
335 | } | |
7bfa5ab6 MS |
336 | |
337 | /* | |
338 | * Support for reserved memory regions defined in device tree | |
339 | */ | |
340 | #ifdef CONFIG_OF_RESERVED_MEM | |
341 | #include <linux/of.h> | |
342 | #include <linux/of_fdt.h> | |
343 | #include <linux/of_reserved_mem.h> | |
344 | ||
93228b44 VM |
345 | static struct reserved_mem *dma_reserved_default_memory __initdata; |
346 | ||
7bfa5ab6 MS |
347 | static int rmem_dma_device_init(struct reserved_mem *rmem, struct device *dev) |
348 | { | |
349 | struct dma_coherent_mem *mem = rmem->priv; | |
d35b0996 AB |
350 | int ret; |
351 | ||
6d573398 AB |
352 | if (!mem) { |
353 | ret = dma_init_coherent_memory(rmem->base, rmem->base, | |
354 | rmem->size, | |
355 | DMA_MEMORY_EXCLUSIVE, &mem); | |
356 | if (ret) { | |
357 | pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n", | |
358 | &rmem->base, (unsigned long)rmem->size / SZ_1M); | |
359 | return ret; | |
360 | } | |
7bfa5ab6 | 361 | } |
c41f9ea9 | 362 | mem->use_dev_dma_pfn_offset = true; |
7bfa5ab6 MS |
363 | rmem->priv = mem; |
364 | dma_assign_coherent_memory(dev, mem); | |
365 | return 0; | |
366 | } | |
367 | ||
368 | static void rmem_dma_device_release(struct reserved_mem *rmem, | |
369 | struct device *dev) | |
370 | { | |
93228b44 VM |
371 | if (dev) |
372 | dev->dma_mem = NULL; | |
7bfa5ab6 MS |
373 | } |
374 | ||
375 | static const struct reserved_mem_ops rmem_dma_ops = { | |
376 | .device_init = rmem_dma_device_init, | |
377 | .device_release = rmem_dma_device_release, | |
378 | }; | |
379 | ||
380 | static int __init rmem_dma_setup(struct reserved_mem *rmem) | |
381 | { | |
382 | unsigned long node = rmem->fdt_node; | |
383 | ||
384 | if (of_get_flat_dt_prop(node, "reusable", NULL)) | |
385 | return -EINVAL; | |
386 | ||
387 | #ifdef CONFIG_ARM | |
388 | if (!of_get_flat_dt_prop(node, "no-map", NULL)) { | |
389 | pr_err("Reserved memory: regions without no-map are not yet supported\n"); | |
390 | return -EINVAL; | |
391 | } | |
93228b44 VM |
392 | |
393 | if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) { | |
394 | WARN(dma_reserved_default_memory, | |
395 | "Reserved memory: region for default DMA coherent area is redefined\n"); | |
396 | dma_reserved_default_memory = rmem; | |
397 | } | |
7bfa5ab6 MS |
398 | #endif |
399 | ||
400 | rmem->ops = &rmem_dma_ops; | |
401 | pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n", | |
402 | &rmem->base, (unsigned long)rmem->size / SZ_1M); | |
403 | return 0; | |
404 | } | |
93228b44 VM |
405 | |
406 | static int __init dma_init_reserved_memory(void) | |
407 | { | |
408 | const struct reserved_mem_ops *ops; | |
409 | int ret; | |
410 | ||
411 | if (!dma_reserved_default_memory) | |
412 | return -ENOMEM; | |
413 | ||
414 | ops = dma_reserved_default_memory->ops; | |
415 | ||
416 | /* | |
417 | * We rely on rmem_dma_device_init() does not propagate error of | |
418 | * dma_assign_coherent_memory() for "NULL" device. | |
419 | */ | |
420 | ret = ops->device_init(dma_reserved_default_memory, NULL); | |
421 | ||
422 | if (!ret) { | |
423 | dma_coherent_default_memory = dma_reserved_default_memory->priv; | |
424 | pr_info("DMA: default coherent area is set\n"); | |
425 | } | |
426 | ||
427 | return ret; | |
428 | } | |
429 | ||
430 | core_initcall(dma_init_reserved_memory); | |
431 | ||
7bfa5ab6 MS |
432 | RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup); |
433 | #endif |