2 * Copyright (C) 2007-2010 Lawrence Livermore National Security, LLC.
3 * Copyright (C) 2007 The Regents of the University of California.
4 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
5 * Written by Brian Behlendorf <behlendorf1@llnl.gov>.
8 * This file is part of the SPL, Solaris Porting Layer.
9 * For details, see <http://zfsonlinux.org/>.
11 * The SPL is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU General Public License as published by the
13 * Free Software Foundation; either version 2 of the License, or (at your
14 * option) any later version.
16 * The SPL is distributed in the hope that it will be useful, but WITHOUT
17 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 * You should have received a copy of the GNU General Public License along
22 * with the SPL. If not, see <http://www.gnu.org/licenses/>.
25 #include <sys/debug.h>
26 #include <sys/sysmacros.h>
30 #include <linux/ratelimit.h>
33 * As a general rule kmem_alloc() allocations should be small, preferably
34 * just a few pages since they must by physically contiguous. Therefore, a
35 * rate limited warning will be printed to the console for any kmem_alloc()
36 * which exceeds a reasonable threshold.
38 * The default warning threshold is set to eight pages but capped at 32K to
39 * accommodate systems using large pages. This value was selected to be small
40 * enough to ensure the largest allocations are quickly noticed and fixed.
41 * But large enough to avoid logging any warnings when a allocation size is
42 * larger than optimal but not a serious concern. Since this value is tunable,
43 * developers are encouraged to set it lower when testing so any new largish
44 * allocations are quickly caught. These warnings may be disabled by setting
45 * the threshold to zero.
47 unsigned int spl_kmem_alloc_warn
= MAX(8 * PAGE_SIZE
, 32 * 1024);
48 module_param(spl_kmem_alloc_warn
, uint
, 0644);
49 MODULE_PARM_DESC(spl_kmem_alloc_warn
,
50 "Warning threshold in bytes for a kmem_alloc()");
51 EXPORT_SYMBOL(spl_kmem_alloc_warn
);
54 * Large kmem_alloc() allocations will fail if they exceed KMALLOC_MAX_SIZE.
55 * Allocations which are marginally smaller than this limit may succeed but
56 * should still be avoided due to the expense of locating a contiguous range
57 * of free pages. Therefore, a maximum kmem size with reasonable safely
58 * margin of 4x is set. Kmem_alloc() allocations larger than this maximum
59 * will quickly fail. Vmem_alloc() allocations less than or equal to this
60 * value will use kmalloc(), but shift to vmalloc() when exceeding this value.
62 unsigned int spl_kmem_alloc_max
= (KMALLOC_MAX_SIZE
>> 2);
63 module_param(spl_kmem_alloc_max
, uint
, 0644);
64 MODULE_PARM_DESC(spl_kmem_alloc_max
,
65 "Maximum size in bytes for a kmem_alloc()");
66 EXPORT_SYMBOL(spl_kmem_alloc_max
);
73 EXPORT_SYMBOL(kmem_debugging
);
76 kmem_vasprintf(const char *fmt
, va_list ap
)
83 ptr
= kvasprintf(GFP_KERNEL
, fmt
, aq
);
85 } while (ptr
== NULL
);
89 EXPORT_SYMBOL(kmem_vasprintf
);
92 kmem_asprintf(const char *fmt
, ...)
99 ptr
= kvasprintf(GFP_KERNEL
, fmt
, ap
);
101 } while (ptr
== NULL
);
105 EXPORT_SYMBOL(kmem_asprintf
);
108 __strdup(const char *str
, int flags
)
114 ptr
= kmalloc(n
+ 1, kmem_flags_convert(flags
));
116 memcpy(ptr
, str
, n
+ 1);
122 strdup(const char *str
)
124 return (__strdup(str
, KM_SLEEP
));
126 EXPORT_SYMBOL(strdup
);
133 EXPORT_SYMBOL(strfree
);
136 * Limit the number of large allocation stack traces dumped to not more than
137 * 5 every 60 seconds to prevent denial-of-service attacks from debug code.
139 DEFINE_RATELIMIT_STATE(kmem_alloc_ratelimit_state
, 60 * HZ
, 5);
142 * General purpose unified implementation of kmem_alloc(). It is an
143 * amalgamation of Linux and Illumos allocator design. It should never be
144 * exported to ensure that code using kmem_alloc()/kmem_zalloc() remains
145 * relatively portable. Consumers may only access this function through
146 * wrappers that enforce the common flags to ensure portability.
149 spl_kmem_alloc_impl(size_t size
, int flags
, int node
)
151 gfp_t lflags
= kmem_flags_convert(flags
);
155 * Log abnormally large allocations and rate limit the console output.
156 * Allocations larger than spl_kmem_alloc_warn should be performed
157 * through the vmem_alloc()/vmem_zalloc() interfaces.
159 if ((spl_kmem_alloc_warn
> 0) && (size
> spl_kmem_alloc_warn
) &&
160 !(flags
& KM_VMEM
) && __ratelimit(&kmem_alloc_ratelimit_state
)) {
162 "Large kmem_alloc(%lu, 0x%x), please file an issue at:\n"
163 "https://github.com/zfsonlinux/zfs/issues/new\n",
164 (unsigned long)size
, flags
);
169 * Use a loop because kmalloc_node() can fail when GFP_KERNEL is used
170 * unlike kmem_alloc() with KM_SLEEP on Illumos.
174 * Calling kmalloc_node() when the size >= spl_kmem_alloc_max
175 * is unsafe. This must fail for all for kmem_alloc() and
176 * kmem_zalloc() callers.
178 * For vmem_alloc() and vmem_zalloc() callers it is permissible
179 * to use __vmalloc(). However, in general use of __vmalloc()
180 * is strongly discouraged because a global lock must be
181 * acquired. Contention on this lock can significantly
182 * impact performance so frequently manipulating the virtual
183 * address space is strongly discouraged.
185 if (unlikely(size
> spl_kmem_alloc_max
)) {
186 if (flags
& KM_VMEM
) {
187 ptr
= __vmalloc(size
, lflags
, PAGE_KERNEL
);
192 ptr
= kmalloc_node(size
, lflags
, node
);
195 if (likely(ptr
) || (flags
& KM_NOSLEEP
))
198 if (unlikely(__ratelimit(&kmem_alloc_ratelimit_state
))) {
200 "Possible memory allocation deadlock: "
201 "size=%lu lflags=0x%x",
202 (unsigned long)size
, lflags
);
207 * Use cond_resched() instead of congestion_wait() to avoid
208 * deadlocking systems where there are no block devices.
217 spl_kmem_free_impl(const void *buf
, size_t size
)
219 if (is_vmalloc_addr(buf
))
226 * Memory allocation and accounting for kmem_* * style allocations. When
227 * DEBUG_KMEM is enabled the total memory allocated will be tracked and
228 * any memory leaked will be reported during module unload.
230 * ./configure --enable-debug-kmem
234 /* Shim layer memory accounting */
235 #ifdef HAVE_ATOMIC64_T
236 atomic64_t kmem_alloc_used
= ATOMIC64_INIT(0);
237 unsigned long long kmem_alloc_max
= 0;
238 #else /* HAVE_ATOMIC64_T */
239 atomic_t kmem_alloc_used
= ATOMIC_INIT(0);
240 unsigned long long kmem_alloc_max
= 0;
241 #endif /* HAVE_ATOMIC64_T */
243 EXPORT_SYMBOL(kmem_alloc_used
);
244 EXPORT_SYMBOL(kmem_alloc_max
);
247 spl_kmem_alloc_debug(size_t size
, int flags
, int node
)
251 ptr
= spl_kmem_alloc_impl(size
, flags
, node
);
253 kmem_alloc_used_add(size
);
254 if (unlikely(kmem_alloc_used_read() > kmem_alloc_max
))
255 kmem_alloc_max
= kmem_alloc_used_read();
262 spl_kmem_free_debug(const void *ptr
, size_t size
)
264 kmem_alloc_used_sub(size
);
265 spl_kmem_free_impl(ptr
, size
);
269 * When DEBUG_KMEM_TRACKING is enabled not only will total bytes be tracked
270 * but also the location of every alloc and free. When the SPL module is
271 * unloaded a list of all leaked addresses and where they were allocated
272 * will be dumped to the console. Enabling this feature has a significant
273 * impact on performance but it makes finding memory leaks straight forward.
275 * Not surprisingly with debugging enabled the xmem_locks are very highly
276 * contended particularly on xfree(). If we want to run with this detailed
277 * debugging enabled for anything other than debugging we need to minimize
278 * the contention by moving to a lock per xmem_table entry model.
280 * ./configure --enable-debug-kmem-tracking
282 #ifdef DEBUG_KMEM_TRACKING
284 #include <linux/hash.h>
285 #include <linux/ctype.h>
287 #define KMEM_HASH_BITS 10
288 #define KMEM_TABLE_SIZE (1 << KMEM_HASH_BITS)
290 typedef struct kmem_debug
{
291 struct hlist_node kd_hlist
; /* Hash node linkage */
292 struct list_head kd_list
; /* List of all allocations */
293 void *kd_addr
; /* Allocation pointer */
294 size_t kd_size
; /* Allocation size */
295 const char *kd_func
; /* Allocation function */
296 int kd_line
; /* Allocation line */
299 static spinlock_t kmem_lock
;
300 static struct hlist_head kmem_table
[KMEM_TABLE_SIZE
];
301 static struct list_head kmem_list
;
303 static kmem_debug_t
*
304 kmem_del_init(spinlock_t
*lock
, struct hlist_head
*table
,
305 int bits
, const void *addr
)
307 struct hlist_head
*head
;
308 struct hlist_node
*node
;
309 struct kmem_debug
*p
;
312 spin_lock_irqsave(lock
, flags
);
314 head
= &table
[hash_ptr((void *)addr
, bits
)];
315 hlist_for_each(node
, head
) {
316 p
= list_entry(node
, struct kmem_debug
, kd_hlist
);
317 if (p
->kd_addr
== addr
) {
318 hlist_del_init(&p
->kd_hlist
);
319 list_del_init(&p
->kd_list
);
320 spin_unlock_irqrestore(lock
, flags
);
325 spin_unlock_irqrestore(lock
, flags
);
331 spl_kmem_alloc_track(size_t size
, int flags
,
332 const char *func
, int line
, int node
)
336 unsigned long irq_flags
;
338 dptr
= kmalloc(sizeof (kmem_debug_t
), kmem_flags_convert(flags
));
342 dptr
->kd_func
= __strdup(func
, flags
);
343 if (dptr
->kd_func
== NULL
) {
348 ptr
= spl_kmem_alloc_debug(size
, flags
, node
);
350 kfree(dptr
->kd_func
);
355 INIT_HLIST_NODE(&dptr
->kd_hlist
);
356 INIT_LIST_HEAD(&dptr
->kd_list
);
359 dptr
->kd_size
= size
;
360 dptr
->kd_line
= line
;
362 spin_lock_irqsave(&kmem_lock
, irq_flags
);
363 hlist_add_head(&dptr
->kd_hlist
,
364 &kmem_table
[hash_ptr(ptr
, KMEM_HASH_BITS
)]);
365 list_add_tail(&dptr
->kd_list
, &kmem_list
);
366 spin_unlock_irqrestore(&kmem_lock
, irq_flags
);
372 spl_kmem_free_track(const void *ptr
, size_t size
)
376 /* Must exist in hash due to kmem_alloc() */
377 dptr
= kmem_del_init(&kmem_lock
, kmem_table
, KMEM_HASH_BITS
, ptr
);
378 ASSERT3P(dptr
, !=, NULL
);
379 ASSERT3S(dptr
->kd_size
, ==, size
);
381 kfree(dptr
->kd_func
);
384 spl_kmem_free_debug(ptr
, size
);
386 #endif /* DEBUG_KMEM_TRACKING */
387 #endif /* DEBUG_KMEM */
390 * Public kmem_alloc(), kmem_zalloc() and kmem_free() interfaces.
393 spl_kmem_alloc(size_t size
, int flags
, const char *func
, int line
)
395 ASSERT0(flags
& ~KM_PUBLIC_MASK
);
397 #if !defined(DEBUG_KMEM)
398 return (spl_kmem_alloc_impl(size
, flags
, NUMA_NO_NODE
));
399 #elif !defined(DEBUG_KMEM_TRACKING)
400 return (spl_kmem_alloc_debug(size
, flags
, NUMA_NO_NODE
));
402 return (spl_kmem_alloc_track(size
, flags
, func
, line
, NUMA_NO_NODE
));
405 EXPORT_SYMBOL(spl_kmem_alloc
);
408 spl_kmem_zalloc(size_t size
, int flags
, const char *func
, int line
)
410 ASSERT0(flags
& ~KM_PUBLIC_MASK
);
414 #if !defined(DEBUG_KMEM)
415 return (spl_kmem_alloc_impl(size
, flags
, NUMA_NO_NODE
));
416 #elif !defined(DEBUG_KMEM_TRACKING)
417 return (spl_kmem_alloc_debug(size
, flags
, NUMA_NO_NODE
));
419 return (spl_kmem_alloc_track(size
, flags
, func
, line
, NUMA_NO_NODE
));
422 EXPORT_SYMBOL(spl_kmem_zalloc
);
425 spl_kmem_free(const void *buf
, size_t size
)
427 #if !defined(DEBUG_KMEM)
428 return (spl_kmem_free_impl(buf
, size
));
429 #elif !defined(DEBUG_KMEM_TRACKING)
430 return (spl_kmem_free_debug(buf
, size
));
432 return (spl_kmem_free_track(buf
, size
));
435 EXPORT_SYMBOL(spl_kmem_free
);
437 #if defined(DEBUG_KMEM) && defined(DEBUG_KMEM_TRACKING)
439 spl_sprintf_addr(kmem_debug_t
*kd
, char *str
, int len
, int min
)
441 int size
= ((len
- 1) < kd
->kd_size
) ? (len
- 1) : kd
->kd_size
;
444 ASSERT(str
!= NULL
&& len
>= 17);
448 * Check for a fully printable string, and while we are at
449 * it place the printable characters in the passed buffer.
451 for (i
= 0; i
< size
; i
++) {
452 str
[i
] = ((char *)(kd
->kd_addr
))[i
];
453 if (isprint(str
[i
])) {
457 * Minimum number of printable characters found
458 * to make it worthwhile to print this as ascii.
469 sprintf(str
, "%02x%02x%02x%02x%02x%02x%02x%02x",
470 *((uint8_t *)kd
->kd_addr
),
471 *((uint8_t *)kd
->kd_addr
+ 2),
472 *((uint8_t *)kd
->kd_addr
+ 4),
473 *((uint8_t *)kd
->kd_addr
+ 6),
474 *((uint8_t *)kd
->kd_addr
+ 8),
475 *((uint8_t *)kd
->kd_addr
+ 10),
476 *((uint8_t *)kd
->kd_addr
+ 12),
477 *((uint8_t *)kd
->kd_addr
+ 14));
484 spl_kmem_init_tracking(struct list_head
*list
, spinlock_t
*lock
, int size
)
488 spin_lock_init(lock
);
489 INIT_LIST_HEAD(list
);
491 for (i
= 0; i
< size
; i
++)
492 INIT_HLIST_HEAD(&kmem_table
[i
]);
498 spl_kmem_fini_tracking(struct list_head
*list
, spinlock_t
*lock
)
504 spin_lock_irqsave(lock
, flags
);
505 if (!list_empty(list
))
506 printk(KERN_WARNING
"%-16s %-5s %-16s %s:%s\n", "address",
507 "size", "data", "func", "line");
509 list_for_each_entry(kd
, list
, kd_list
)
510 printk(KERN_WARNING
"%p %-5d %-16s %s:%d\n", kd
->kd_addr
,
511 (int)kd
->kd_size
, spl_sprintf_addr(kd
, str
, 17, 8),
512 kd
->kd_func
, kd
->kd_line
);
514 spin_unlock_irqrestore(lock
, flags
);
516 #endif /* DEBUG_KMEM && DEBUG_KMEM_TRACKING */
522 kmem_alloc_used_set(0);
524 #ifdef DEBUG_KMEM_TRACKING
525 spl_kmem_init_tracking(&kmem_list
, &kmem_lock
, KMEM_TABLE_SIZE
);
526 #endif /* DEBUG_KMEM_TRACKING */
527 #endif /* DEBUG_KMEM */
537 * Display all unreclaimed memory addresses, including the
538 * allocation size and the first few bytes of what's located
539 * at that address to aid in debugging. Performance is not
540 * a serious concern here since it is module unload time.
542 if (kmem_alloc_used_read() != 0)
543 printk(KERN_WARNING
"kmem leaked %ld/%llu bytes\n",
544 (unsigned long)kmem_alloc_used_read(), kmem_alloc_max
);
546 #ifdef DEBUG_KMEM_TRACKING
547 spl_kmem_fini_tracking(&kmem_list
, &kmem_lock
);
548 #endif /* DEBUG_KMEM_TRACKING */
549 #endif /* DEBUG_KMEM */