2 * Coherent per-device memory handling.
6 #include <linux/slab.h>
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/dma-mapping.h>
11 struct dma_coherent_mem
{
13 dma_addr_t device_base
;
14 unsigned long pfn_base
;
17 unsigned long *bitmap
;
19 bool use_dev_dma_pfn_offset
;
22 static struct dma_coherent_mem
*dma_coherent_default_memory __ro_after_init
;
24 static inline struct dma_coherent_mem
*dev_get_coherent_memory(struct device
*dev
)
26 if (dev
&& dev
->dma_mem
)
31 static inline dma_addr_t
dma_get_device_base(struct device
*dev
,
32 struct dma_coherent_mem
* mem
)
34 if (mem
->use_dev_dma_pfn_offset
)
35 return (mem
->pfn_base
- dev
->dma_pfn_offset
) << PAGE_SHIFT
;
37 return mem
->device_base
;
40 static bool dma_init_coherent_memory(
41 phys_addr_t phys_addr
, dma_addr_t device_addr
, size_t size
, int flags
,
42 struct dma_coherent_mem
**mem
)
44 struct dma_coherent_mem
*dma_mem
= NULL
;
45 void __iomem
*mem_base
= NULL
;
46 int pages
= size
>> PAGE_SHIFT
;
47 int bitmap_size
= BITS_TO_LONGS(pages
) * sizeof(long);
49 if ((flags
& (DMA_MEMORY_MAP
| DMA_MEMORY_IO
)) == 0)
54 if (flags
& DMA_MEMORY_MAP
)
55 mem_base
= memremap(phys_addr
, size
, MEMREMAP_WC
);
57 mem_base
= ioremap(phys_addr
, size
);
61 dma_mem
= kzalloc(sizeof(struct dma_coherent_mem
), GFP_KERNEL
);
64 dma_mem
->bitmap
= kzalloc(bitmap_size
, GFP_KERNEL
);
68 dma_mem
->virt_base
= mem_base
;
69 dma_mem
->device_base
= device_addr
;
70 dma_mem
->pfn_base
= PFN_DOWN(phys_addr
);
71 dma_mem
->size
= pages
;
72 dma_mem
->flags
= flags
;
73 spin_lock_init(&dma_mem
->spinlock
);
81 if (flags
& DMA_MEMORY_MAP
)
89 static void dma_release_coherent_memory(struct dma_coherent_mem
*mem
)
94 if (mem
->flags
& DMA_MEMORY_MAP
)
95 memunmap(mem
->virt_base
);
97 iounmap(mem
->virt_base
);
102 static int dma_assign_coherent_memory(struct device
*dev
,
103 struct dma_coherent_mem
*mem
)
112 /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
117 int dma_declare_coherent_memory(struct device
*dev
, phys_addr_t phys_addr
,
118 dma_addr_t device_addr
, size_t size
, int flags
)
120 struct dma_coherent_mem
*mem
;
122 if (!dma_init_coherent_memory(phys_addr
, device_addr
, size
, flags
,
126 if (dma_assign_coherent_memory(dev
, mem
) == 0)
127 return flags
& DMA_MEMORY_MAP
? DMA_MEMORY_MAP
: DMA_MEMORY_IO
;
129 dma_release_coherent_memory(mem
);
132 EXPORT_SYMBOL(dma_declare_coherent_memory
);
134 void dma_release_declared_memory(struct device
*dev
)
136 struct dma_coherent_mem
*mem
= dev
->dma_mem
;
140 dma_release_coherent_memory(mem
);
143 EXPORT_SYMBOL(dma_release_declared_memory
);
145 void *dma_mark_declared_memory_occupied(struct device
*dev
,
146 dma_addr_t device_addr
, size_t size
)
148 struct dma_coherent_mem
*mem
= dev
->dma_mem
;
152 size
+= device_addr
& ~PAGE_MASK
;
155 return ERR_PTR(-EINVAL
);
157 spin_lock_irqsave(&mem
->spinlock
, flags
);
158 pos
= PFN_DOWN(device_addr
- dma_get_device_base(dev
, mem
));
159 err
= bitmap_allocate_region(mem
->bitmap
, pos
, get_order(size
));
160 spin_unlock_irqrestore(&mem
->spinlock
, flags
);
164 return mem
->virt_base
+ (pos
<< PAGE_SHIFT
);
166 EXPORT_SYMBOL(dma_mark_declared_memory_occupied
);
168 static void *__dma_alloc_from_coherent(struct dma_coherent_mem
*mem
,
169 ssize_t size
, dma_addr_t
*dma_handle
)
171 int order
= get_order(size
);
177 spin_lock_irqsave(&mem
->spinlock
, flags
);
179 if (unlikely(size
> (mem
->size
<< PAGE_SHIFT
)))
182 pageno
= bitmap_find_free_region(mem
->bitmap
, mem
->size
, order
);
183 if (unlikely(pageno
< 0))
187 * Memory was found in the coherent area.
189 *dma_handle
= mem
->device_base
+ (pageno
<< PAGE_SHIFT
);
190 ret
= mem
->virt_base
+ (pageno
<< PAGE_SHIFT
);
191 dma_memory_map
= (mem
->flags
& DMA_MEMORY_MAP
);
192 spin_unlock_irqrestore(&mem
->spinlock
, flags
);
194 memset(ret
, 0, size
);
196 memset_io(ret
, 0, size
);
201 spin_unlock_irqrestore(&mem
->spinlock
, flags
);
206 * dma_alloc_from_dev_coherent() - allocate memory from device coherent pool
207 * @dev: device from which we allocate memory
208 * @size: size of requested memory area
209 * @dma_handle: This will be filled with the correct dma handle
210 * @ret: This pointer will be filled with the virtual address
213 * This function should be only called from per-arch dma_alloc_coherent()
214 * to support allocation from per-device coherent memory pools.
216 * Returns 0 if dma_alloc_coherent should continue with allocating from
217 * generic memory areas, or !0 if dma_alloc_coherent should return @ret.
219 int dma_alloc_from_dev_coherent(struct device
*dev
, ssize_t size
,
220 dma_addr_t
*dma_handle
, void **ret
)
222 struct dma_coherent_mem
*mem
= dev_get_coherent_memory(dev
);
227 *ret
= __dma_alloc_from_coherent(mem
, size
, dma_handle
);
232 * In the case where the allocation can not be satisfied from the
233 * per-device area, try to fall back to generic memory if the
234 * constraints allow it.
236 return mem
->flags
& DMA_MEMORY_EXCLUSIVE
;
238 EXPORT_SYMBOL(dma_alloc_from_dev_coherent
);
240 void *dma_alloc_from_global_coherent(ssize_t size
, dma_addr_t
*dma_handle
)
242 if (!dma_coherent_default_memory
)
245 return __dma_alloc_from_coherent(dma_coherent_default_memory
, size
,
249 static int __dma_release_from_coherent(struct dma_coherent_mem
*mem
,
250 int order
, void *vaddr
)
252 if (mem
&& vaddr
>= mem
->virt_base
&& vaddr
<
253 (mem
->virt_base
+ (mem
->size
<< PAGE_SHIFT
))) {
254 int page
= (vaddr
- mem
->virt_base
) >> PAGE_SHIFT
;
257 spin_lock_irqsave(&mem
->spinlock
, flags
);
258 bitmap_release_region(mem
->bitmap
, page
, order
);
259 spin_unlock_irqrestore(&mem
->spinlock
, flags
);
266 * dma_release_from_dev_coherent() - free memory to device coherent memory pool
267 * @dev: device from which the memory was allocated
268 * @order: the order of pages allocated
269 * @vaddr: virtual address of allocated pages
271 * This checks whether the memory was allocated from the per-device
272 * coherent memory pool and if so, releases that memory.
274 * Returns 1 if we correctly released the memory, or 0 if the caller should
275 * proceed with releasing memory from generic pools.
277 int dma_release_from_dev_coherent(struct device
*dev
, int order
, void *vaddr
)
279 struct dma_coherent_mem
*mem
= dev_get_coherent_memory(dev
);
281 return __dma_release_from_coherent(mem
, order
, vaddr
);
283 EXPORT_SYMBOL(dma_release_from_dev_coherent
);
285 int dma_release_from_global_coherent(int order
, void *vaddr
)
287 if (!dma_coherent_default_memory
)
290 return __dma_release_from_coherent(dma_coherent_default_memory
, order
,
294 static int __dma_mmap_from_coherent(struct dma_coherent_mem
*mem
,
295 struct vm_area_struct
*vma
, void *vaddr
, size_t size
, int *ret
)
297 if (mem
&& vaddr
>= mem
->virt_base
&& vaddr
+ size
<=
298 (mem
->virt_base
+ (mem
->size
<< PAGE_SHIFT
))) {
299 unsigned long off
= vma
->vm_pgoff
;
300 int start
= (vaddr
- mem
->virt_base
) >> PAGE_SHIFT
;
301 int user_count
= vma_pages(vma
);
302 int count
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
305 if (off
< count
&& user_count
<= count
- off
) {
306 unsigned long pfn
= mem
->pfn_base
+ start
+ off
;
307 *ret
= remap_pfn_range(vma
, vma
->vm_start
, pfn
,
308 user_count
<< PAGE_SHIFT
,
317 * dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool
318 * @dev: device from which the memory was allocated
319 * @vma: vm_area for the userspace memory
320 * @vaddr: cpu address returned by dma_alloc_from_dev_coherent
321 * @size: size of the memory buffer allocated
322 * @ret: result from remap_pfn_range()
324 * This checks whether the memory was allocated from the per-device
325 * coherent memory pool and if so, maps that memory to the provided vma.
327 * Returns 1 if we correctly mapped the memory, or 0 if the caller should
328 * proceed with mapping memory from generic pools.
330 int dma_mmap_from_dev_coherent(struct device
*dev
, struct vm_area_struct
*vma
,
331 void *vaddr
, size_t size
, int *ret
)
333 struct dma_coherent_mem
*mem
= dev_get_coherent_memory(dev
);
335 return __dma_mmap_from_coherent(mem
, vma
, vaddr
, size
, ret
);
337 EXPORT_SYMBOL(dma_mmap_from_dev_coherent
);
339 int dma_mmap_from_global_coherent(struct vm_area_struct
*vma
, void *vaddr
,
340 size_t size
, int *ret
)
342 if (!dma_coherent_default_memory
)
345 return __dma_mmap_from_coherent(dma_coherent_default_memory
, vma
,
350 * Support for reserved memory regions defined in device tree
352 #ifdef CONFIG_OF_RESERVED_MEM
353 #include <linux/of.h>
354 #include <linux/of_fdt.h>
355 #include <linux/of_reserved_mem.h>
357 static struct reserved_mem
*dma_reserved_default_memory __initdata
;
359 static int rmem_dma_device_init(struct reserved_mem
*rmem
, struct device
*dev
)
361 struct dma_coherent_mem
*mem
= rmem
->priv
;
364 !dma_init_coherent_memory(rmem
->base
, rmem
->base
, rmem
->size
,
365 DMA_MEMORY_MAP
| DMA_MEMORY_EXCLUSIVE
,
367 pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",
368 &rmem
->base
, (unsigned long)rmem
->size
/ SZ_1M
);
371 mem
->use_dev_dma_pfn_offset
= true;
373 dma_assign_coherent_memory(dev
, mem
);
377 static void rmem_dma_device_release(struct reserved_mem
*rmem
,
384 static const struct reserved_mem_ops rmem_dma_ops
= {
385 .device_init
= rmem_dma_device_init
,
386 .device_release
= rmem_dma_device_release
,
389 static int __init
rmem_dma_setup(struct reserved_mem
*rmem
)
391 unsigned long node
= rmem
->fdt_node
;
393 if (of_get_flat_dt_prop(node
, "reusable", NULL
))
397 if (!of_get_flat_dt_prop(node
, "no-map", NULL
)) {
398 pr_err("Reserved memory: regions without no-map are not yet supported\n");
402 if (of_get_flat_dt_prop(node
, "linux,dma-default", NULL
)) {
403 WARN(dma_reserved_default_memory
,
404 "Reserved memory: region for default DMA coherent area is redefined\n");
405 dma_reserved_default_memory
= rmem
;
409 rmem
->ops
= &rmem_dma_ops
;
410 pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n",
411 &rmem
->base
, (unsigned long)rmem
->size
/ SZ_1M
);
415 static int __init
dma_init_reserved_memory(void)
417 const struct reserved_mem_ops
*ops
;
420 if (!dma_reserved_default_memory
)
423 ops
= dma_reserved_default_memory
->ops
;
426 * We rely on rmem_dma_device_init() does not propagate error of
427 * dma_assign_coherent_memory() for "NULL" device.
429 ret
= ops
->device_init(dma_reserved_default_memory
, NULL
);
432 dma_coherent_default_memory
= dma_reserved_default_memory
->priv
;
433 pr_info("DMA: default coherent area is set\n");
439 core_initcall(dma_init_reserved_memory
);
441 RESERVEDMEM_OF_DECLARE(dma
, "shared-dma-pool", rmem_dma_setup
);