]>
git.proxmox.com Git - mirror_ubuntu-zesty-kernel.git/blob - mm/dmapool.c
2 #include <linux/device.h>
4 #include <asm/io.h> /* Needed for i386 to build */
5 #include <linux/dma-mapping.h>
6 #include <linux/dmapool.h>
7 #include <linux/slab.h>
8 #include <linux/module.h>
9 #include <linux/poison.h>
10 #include <linux/sched.h>
13 * Pool allocator ... wraps the dma_alloc_coherent page allocator, so
14 * small blocks are easily used by drivers for bus mastering controllers.
15 * This should probably be sharing the guts of the slab allocator.
18 struct dma_pool
{ /* the pool */
19 struct list_head page_list
;
21 size_t blocks_per_page
;
26 wait_queue_head_t waitq
;
27 struct list_head pools
;
30 struct dma_page
{ /* cacheable header for 'allocation' bytes */
31 struct list_head page_list
;
35 unsigned long bitmap
[0];
38 #define POOL_TIMEOUT_JIFFIES ((100 /* msec */ * HZ) / 1000)
40 static DEFINE_MUTEX (pools_lock
);
43 show_pools (struct device
*dev
, struct device_attribute
*attr
, char *buf
)
48 struct dma_page
*page
;
49 struct dma_pool
*pool
;
54 temp
= scnprintf(next
, size
, "poolinfo - 0.1\n");
58 mutex_lock(&pools_lock
);
59 list_for_each_entry(pool
, &dev
->dma_pools
, pools
) {
63 list_for_each_entry(page
, &pool
->page_list
, page_list
) {
65 blocks
+= page
->in_use
;
68 /* per-pool info, no real statistics yet */
69 temp
= scnprintf(next
, size
, "%-16s %4u %4Zu %4Zu %2u\n",
71 blocks
, pages
* pool
->blocks_per_page
,
76 mutex_unlock(&pools_lock
);
78 return PAGE_SIZE
- size
;
80 static DEVICE_ATTR (pools
, S_IRUGO
, show_pools
, NULL
);
83 * dma_pool_create - Creates a pool of consistent memory blocks, for dma.
84 * @name: name of pool, for diagnostics
85 * @dev: device that will be doing the DMA
86 * @size: size of the blocks in this pool.
87 * @align: alignment requirement for blocks; must be a power of two
88 * @allocation: returned blocks won't cross this boundary (or zero)
89 * Context: !in_interrupt()
91 * Returns a dma allocation pool with the requested characteristics, or
92 * null if one can't be created. Given one of these pools, dma_pool_alloc()
93 * may be used to allocate memory. Such memory will all have "consistent"
94 * DMA mappings, accessible by the device and its driver without using
95 * cache flushing primitives. The actual size of blocks allocated may be
96 * larger than requested because of alignment.
98 * If allocation is nonzero, objects returned from dma_pool_alloc() won't
99 * cross that size boundary. This is useful for devices which have
100 * addressing restrictions on individual DMA transfers, such as not crossing
101 * boundaries of 4KBytes.
104 dma_pool_create (const char *name
, struct device
*dev
,
105 size_t size
, size_t align
, size_t allocation
)
107 struct dma_pool
*retval
;
113 else if (size
< align
)
115 else if ((size
% align
) != 0) {
117 size
&= ~(align
- 1);
120 if (allocation
== 0) {
121 if (PAGE_SIZE
< size
)
124 allocation
= PAGE_SIZE
;
125 // FIXME: round up for less fragmentation
126 } else if (allocation
< size
)
129 if (!(retval
= kmalloc_node (sizeof *retval
, GFP_KERNEL
, dev_to_node(dev
))))
132 strlcpy (retval
->name
, name
, sizeof retval
->name
);
136 INIT_LIST_HEAD (&retval
->page_list
);
137 spin_lock_init (&retval
->lock
);
139 retval
->allocation
= allocation
;
140 retval
->blocks_per_page
= allocation
/ size
;
141 init_waitqueue_head (&retval
->waitq
);
146 mutex_lock(&pools_lock
);
147 if (list_empty (&dev
->dma_pools
))
148 ret
= device_create_file (dev
, &dev_attr_pools
);
151 /* note: not currently insisting "name" be unique */
153 list_add (&retval
->pools
, &dev
->dma_pools
);
158 mutex_unlock(&pools_lock
);
160 INIT_LIST_HEAD (&retval
->pools
);
166 static struct dma_page
*
167 pool_alloc_page (struct dma_pool
*pool
, gfp_t mem_flags
)
169 struct dma_page
*page
;
172 mapsize
= pool
->blocks_per_page
;
173 mapsize
= (mapsize
+ BITS_PER_LONG
- 1) / BITS_PER_LONG
;
174 mapsize
*= sizeof (long);
176 page
= kmalloc(mapsize
+ sizeof *page
, mem_flags
);
179 page
->vaddr
= dma_alloc_coherent (pool
->dev
,
184 memset (page
->bitmap
, 0xff, mapsize
); // bit set == free
185 #ifdef CONFIG_DEBUG_SLAB
186 memset (page
->vaddr
, POOL_POISON_FREED
, pool
->allocation
);
188 list_add (&page
->page_list
, &pool
->page_list
);
199 is_page_busy (int blocks
, unsigned long *bitmap
)
202 if (*bitmap
++ != ~0UL)
204 blocks
-= BITS_PER_LONG
;
210 pool_free_page (struct dma_pool
*pool
, struct dma_page
*page
)
212 dma_addr_t dma
= page
->dma
;
214 #ifdef CONFIG_DEBUG_SLAB
215 memset (page
->vaddr
, POOL_POISON_FREED
, pool
->allocation
);
217 dma_free_coherent (pool
->dev
, pool
->allocation
, page
->vaddr
, dma
);
218 list_del (&page
->page_list
);
224 * dma_pool_destroy - destroys a pool of dma memory blocks.
225 * @pool: dma pool that will be destroyed
226 * Context: !in_interrupt()
228 * Caller guarantees that no more memory from the pool is in use,
229 * and that nothing will try to use the pool after this call.
232 dma_pool_destroy (struct dma_pool
*pool
)
234 mutex_lock(&pools_lock
);
235 list_del (&pool
->pools
);
236 if (pool
->dev
&& list_empty (&pool
->dev
->dma_pools
))
237 device_remove_file (pool
->dev
, &dev_attr_pools
);
238 mutex_unlock(&pools_lock
);
240 while (!list_empty (&pool
->page_list
)) {
241 struct dma_page
*page
;
242 page
= list_entry (pool
->page_list
.next
,
243 struct dma_page
, page_list
);
244 if (is_page_busy (pool
->blocks_per_page
, page
->bitmap
)) {
246 dev_err(pool
->dev
, "dma_pool_destroy %s, %p busy\n",
247 pool
->name
, page
->vaddr
);
249 printk (KERN_ERR
"dma_pool_destroy %s, %p busy\n",
250 pool
->name
, page
->vaddr
);
251 /* leak the still-in-use consistent memory */
252 list_del (&page
->page_list
);
255 pool_free_page (pool
, page
);
263 * dma_pool_alloc - get a block of consistent memory
264 * @pool: dma pool that will produce the block
265 * @mem_flags: GFP_* bitmask
266 * @handle: pointer to dma address of block
268 * This returns the kernel virtual address of a currently unused block,
269 * and reports its dma address through the handle.
270 * If such a memory block can't be allocated, null is returned.
273 dma_pool_alloc (struct dma_pool
*pool
, gfp_t mem_flags
, dma_addr_t
*handle
)
276 struct dma_page
*page
;
282 spin_lock_irqsave (&pool
->lock
, flags
);
283 list_for_each_entry(page
, &pool
->page_list
, page_list
) {
285 /* only cachable accesses here ... */
287 i
< pool
->blocks_per_page
;
288 i
+= BITS_PER_LONG
, map
++) {
289 if (page
->bitmap
[map
] == 0)
291 block
= ffz (~ page
->bitmap
[map
]);
292 if ((i
+ block
) < pool
->blocks_per_page
) {
293 clear_bit (block
, &page
->bitmap
[map
]);
294 offset
= (BITS_PER_LONG
* map
) + block
;
295 offset
*= pool
->size
;
300 if (!(page
= pool_alloc_page (pool
, GFP_ATOMIC
))) {
301 if (mem_flags
& __GFP_WAIT
) {
302 DECLARE_WAITQUEUE (wait
, current
);
304 __set_current_state(TASK_INTERRUPTIBLE
);
305 add_wait_queue (&pool
->waitq
, &wait
);
306 spin_unlock_irqrestore (&pool
->lock
, flags
);
308 schedule_timeout (POOL_TIMEOUT_JIFFIES
);
310 remove_wait_queue (&pool
->waitq
, &wait
);
317 clear_bit (0, &page
->bitmap
[0]);
321 retval
= offset
+ page
->vaddr
;
322 *handle
= offset
+ page
->dma
;
323 #ifdef CONFIG_DEBUG_SLAB
324 memset (retval
, POOL_POISON_ALLOCATED
, pool
->size
);
327 spin_unlock_irqrestore (&pool
->lock
, flags
);
332 static struct dma_page
*
333 pool_find_page (struct dma_pool
*pool
, dma_addr_t dma
)
336 struct dma_page
*page
;
338 spin_lock_irqsave (&pool
->lock
, flags
);
339 list_for_each_entry(page
, &pool
->page_list
, page_list
) {
342 if (dma
< (page
->dma
+ pool
->allocation
))
347 spin_unlock_irqrestore (&pool
->lock
, flags
);
353 * dma_pool_free - put block back into dma pool
354 * @pool: the dma pool holding the block
355 * @vaddr: virtual address of block
356 * @dma: dma address of block
358 * Caller promises neither device nor driver will again touch this block
359 * unless it is first re-allocated.
362 dma_pool_free (struct dma_pool
*pool
, void *vaddr
, dma_addr_t dma
)
364 struct dma_page
*page
;
368 if ((page
= pool_find_page(pool
, dma
)) == NULL
) {
370 dev_err(pool
->dev
, "dma_pool_free %s, %p/%lx (bad dma)\n",
371 pool
->name
, vaddr
, (unsigned long) dma
);
373 printk (KERN_ERR
"dma_pool_free %s, %p/%lx (bad dma)\n",
374 pool
->name
, vaddr
, (unsigned long) dma
);
378 block
= dma
- page
->dma
;
380 map
= block
/ BITS_PER_LONG
;
381 block
%= BITS_PER_LONG
;
383 #ifdef CONFIG_DEBUG_SLAB
384 if (((dma
- page
->dma
) + (void *)page
->vaddr
) != vaddr
) {
386 dev_err(pool
->dev
, "dma_pool_free %s, %p (bad vaddr)/%Lx\n",
387 pool
->name
, vaddr
, (unsigned long long) dma
);
389 printk (KERN_ERR
"dma_pool_free %s, %p (bad vaddr)/%Lx\n",
390 pool
->name
, vaddr
, (unsigned long long) dma
);
393 if (page
->bitmap
[map
] & (1UL << block
)) {
395 dev_err(pool
->dev
, "dma_pool_free %s, dma %Lx already free\n",
396 pool
->name
, (unsigned long long)dma
);
398 printk (KERN_ERR
"dma_pool_free %s, dma %Lx already free\n",
399 pool
->name
, (unsigned long long)dma
);
402 memset (vaddr
, POOL_POISON_FREED
, pool
->size
);
405 spin_lock_irqsave (&pool
->lock
, flags
);
407 set_bit (block
, &page
->bitmap
[map
]);
408 if (waitqueue_active (&pool
->waitq
))
409 wake_up (&pool
->waitq
);
411 * Resist a temptation to do
412 * if (!is_page_busy(bpp, page->bitmap)) pool_free_page(pool, page);
413 * Better have a few empty pages hang around.
415 spin_unlock_irqrestore (&pool
->lock
, flags
);
421 static void dmam_pool_release(struct device
*dev
, void *res
)
423 struct dma_pool
*pool
= *(struct dma_pool
**)res
;
425 dma_pool_destroy(pool
);
428 static int dmam_pool_match(struct device
*dev
, void *res
, void *match_data
)
430 return *(struct dma_pool
**)res
== match_data
;
434 * dmam_pool_create - Managed dma_pool_create()
435 * @name: name of pool, for diagnostics
436 * @dev: device that will be doing the DMA
437 * @size: size of the blocks in this pool.
438 * @align: alignment requirement for blocks; must be a power of two
439 * @allocation: returned blocks won't cross this boundary (or zero)
441 * Managed dma_pool_create(). DMA pool created with this function is
442 * automatically destroyed on driver detach.
444 struct dma_pool
*dmam_pool_create(const char *name
, struct device
*dev
,
445 size_t size
, size_t align
, size_t allocation
)
447 struct dma_pool
**ptr
, *pool
;
449 ptr
= devres_alloc(dmam_pool_release
, sizeof(*ptr
), GFP_KERNEL
);
453 pool
= *ptr
= dma_pool_create(name
, dev
, size
, align
, allocation
);
455 devres_add(dev
, ptr
);
463 * dmam_pool_destroy - Managed dma_pool_destroy()
464 * @pool: dma pool that will be destroyed
466 * Managed dma_pool_destroy().
468 void dmam_pool_destroy(struct dma_pool
*pool
)
470 struct device
*dev
= pool
->dev
;
472 dma_pool_destroy(pool
);
473 WARN_ON(devres_destroy(dev
, dmam_pool_release
, dmam_pool_match
, pool
));
476 EXPORT_SYMBOL (dma_pool_create
);
477 EXPORT_SYMBOL (dma_pool_destroy
);
478 EXPORT_SYMBOL (dma_pool_alloc
);
479 EXPORT_SYMBOL (dma_pool_free
);
480 EXPORT_SYMBOL (dmam_pool_create
);
481 EXPORT_SYMBOL (dmam_pool_destroy
);